From a749d71e4520ce90456c8cd31ecc815e3582a25b Mon Sep 17 00:00:00 2001 From: liuxinwei Date: Wed, 5 Jun 2024 08:53:19 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=96=87=E6=A1=A3=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=20softmax=20=E9=87=8F=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/chaos/tutorials/index.md | 1 - doc/chaos/tutorials/relay/index.md | 1 - doc/chaos/tutorials/transform/index.md | 11 - doc/conf.py | 45 +- doc/read/ffi/relay-expr.ipynb | 4 +- doc/read/index.md | 1 + doc/read/qnn/FTVMQnnCanonicalize.ipynb | 209 ++++ doc/read/qnn/FTVMQnnLegalize.ipynb | 187 +++ doc/read/qnn/index.md | 7 + doc/read/qnn/softmax.ipynb | 313 +++++ doc/read/qnn/testing.py | 60 + doc/read/relay/quant/SimulatedQuantize.ipynb | 116 ++ doc/read/relay/quant/annotate.ipynb | 169 ++- doc/read/relay/quant/calibrate.ipynb | 434 +++++++ doc/read/relay/quant/index.md | 3 + doc/read/relay/quant/partition.ipynb | 86 +- .../quant/realize/CastDtypeInputRealize.ipynb | 52 + .../GetFixedPointMultiplierShift.ipynb | 168 +++ doc/read/relay/quant/realize/MulAndDiv.ipynb | 76 ++ .../relay/quant/realize/UnifyDTypeScale.ipynb | 146 +++ doc/read/relay/quant/realize/common.ipynb | 188 +++ doc/read/relay/quant/realize/index.md | 9 + doc/read/tir/q_multiply_shift/intro.ipynb | 83 +- .../transforms}/SimplifyInference.ipynb | 0 .../{ => chaos}/InferTypeLocal.ipynb | 0 .../transforms/{ => chaos}/defuse-ops.ipynb | 0 .../transforms/{ => chaos}/div-to-mul.ipynb | 0 .../transforms/chaos}/function.ipynb | 0 .../transforms/chaos}/fuse-ops.ipynb | 0 .../transforms/chaos}/index.md | 9 +- .../transforms/chaos}/instrument.ipynb | 0 .../transforms/chaos}/module.ipynb | 0 .../transforms/chaos}/pass.ipynb | 0 .../transforms/chaos}/print-ir.ipynb | 0 .../transforms/chaos}/sequential.ipynb | 0 .../transforms/chaos}/utils/helper.py | 0 .../transforms}/custom-pass.ipynb | 0 doc/read/transforms/index.md | 13 +- .../transform => read/transforms}/infra.ipynb | 2 +- .../transform => read/transforms}/intro.md | 0 .../transform => read/transforms}/pass.ipynb | 0 doc/read/transforms/set_env.py | 6 - .../configs/create_config.py | 0 .../{auto-quantize => aq}/configs/model.toml | 0 .../{auto-quantize => aq}/configs/set_env.py | 0 .../{auto-quantize => aq}/configs/set_tool.py | 0 .../{auto-quantize => aq}/custom.ipynb | 0 .../fixed-point-multiply.ipynb | 0 doc/tutorials/aq/images/ap-intro.png | Bin 0 -> 119442 bytes doc/tutorials/{auto-quantize => aq}/index.md | 4 +- doc/tutorials/aq/intro.ipynb | 124 ++ .../{auto-quantize => aq}/parse.ipynb | 0 .../{auto-quantize => aq}/test.ipynb | 0 doc/tutorials/{auto-quantize => aq}/test.txt | 0 .../{auto-quantize => aq}/tools/__init_.py | 0 .../{auto-quantize => aq}/tools/common.py | 0 .../tools/pattern/__init_.py | 0 .../tools/pattern/common.py | 0 .../tools/pattern/float.py | 0 .../tools/pattern/quant.py | 0 .../fixed-point-multiply-cpp.ipynb | 117 -- doc/tutorials/index.md | 2 +- doc/tutorials/intro/custom-op.ipynb | 58 +- doc/tutorials/intro/custom-quant-op.ipynb | 842 ------------- doc/tutorials/intro/index.md | 2 +- doc/tutorials/intro/rewrite-quant-op.ipynb | 1060 +++++++++++++++++ src/tvm_book/special/rewriter/softmax.py | 19 - 67 files changed, 3474 insertions(+), 1153 deletions(-) delete mode 100755 doc/chaos/tutorials/transform/index.md create mode 100644 doc/read/qnn/FTVMQnnCanonicalize.ipynb create mode 100644 doc/read/qnn/FTVMQnnLegalize.ipynb create mode 100644 doc/read/qnn/index.md create mode 100644 doc/read/qnn/softmax.ipynb create mode 100644 doc/read/qnn/testing.py create mode 100644 doc/read/relay/quant/SimulatedQuantize.ipynb create mode 100644 doc/read/relay/quant/calibrate.ipynb create mode 100644 doc/read/relay/quant/realize/CastDtypeInputRealize.ipynb create mode 100644 doc/read/relay/quant/realize/GetFixedPointMultiplierShift.ipynb create mode 100644 doc/read/relay/quant/realize/MulAndDiv.ipynb create mode 100644 doc/read/relay/quant/realize/UnifyDTypeScale.ipynb create mode 100644 doc/read/relay/quant/realize/common.ipynb create mode 100644 doc/read/relay/quant/realize/index.md rename doc/{chaos/tutorials/transform => read/transforms}/SimplifyInference.ipynb (100%) rename doc/read/transforms/{ => chaos}/InferTypeLocal.ipynb (100%) rename doc/read/transforms/{ => chaos}/defuse-ops.ipynb (100%) rename doc/read/transforms/{ => chaos}/div-to-mul.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/function.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/fuse-ops.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/index.md (55%) mode change 100755 => 100644 rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/instrument.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/module.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/pass.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/print-ir.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/sequential.ipynb (100%) rename doc/{chaos/tutorials/relay/transform => read/transforms/chaos}/utils/helper.py (100%) rename doc/{chaos/tutorials/transform => read/transforms}/custom-pass.ipynb (100%) mode change 100644 => 100755 doc/read/transforms/index.md rename doc/{chaos/tutorials/transform => read/transforms}/infra.ipynb (95%) rename doc/{chaos/tutorials/transform => read/transforms}/intro.md (100%) rename doc/{chaos/tutorials/transform => read/transforms}/pass.ipynb (100%) delete mode 100644 doc/read/transforms/set_env.py rename doc/tutorials/{auto-quantize => aq}/configs/create_config.py (100%) rename doc/tutorials/{auto-quantize => aq}/configs/model.toml (100%) rename doc/tutorials/{auto-quantize => aq}/configs/set_env.py (100%) rename doc/tutorials/{auto-quantize => aq}/configs/set_tool.py (100%) rename doc/tutorials/{auto-quantize => aq}/custom.ipynb (100%) rename doc/tutorials/{auto-quantize => aq}/fixed-point-multiply.ipynb (100%) create mode 100644 doc/tutorials/aq/images/ap-intro.png rename doc/tutorials/{auto-quantize => aq}/index.md (61%) create mode 100644 doc/tutorials/aq/intro.ipynb rename doc/tutorials/{auto-quantize => aq}/parse.ipynb (100%) rename doc/tutorials/{auto-quantize => aq}/test.ipynb (100%) rename doc/tutorials/{auto-quantize => aq}/test.txt (100%) rename doc/tutorials/{auto-quantize => aq}/tools/__init_.py (100%) rename doc/tutorials/{auto-quantize => aq}/tools/common.py (100%) rename doc/tutorials/{auto-quantize => aq}/tools/pattern/__init_.py (100%) rename doc/tutorials/{auto-quantize => aq}/tools/pattern/common.py (100%) rename doc/tutorials/{auto-quantize => aq}/tools/pattern/float.py (100%) rename doc/tutorials/{auto-quantize => aq}/tools/pattern/quant.py (100%) delete mode 100644 doc/tutorials/auto-quantize/fixed-point-multiply-cpp.ipynb delete mode 100644 doc/tutorials/intro/custom-quant-op.ipynb create mode 100644 doc/tutorials/intro/rewrite-quant-op.ipynb diff --git a/doc/chaos/tutorials/index.md b/doc/chaos/tutorials/index.md index 52cea132..b1fd29dc 100755 --- a/doc/chaos/tutorials/index.md +++ b/doc/chaos/tutorials/index.md @@ -7,7 +7,6 @@ :maxdepth: 3 basic/index -transform/index ../quantize/index relay/index ../vta/index diff --git a/doc/chaos/tutorials/relay/index.md b/doc/chaos/tutorials/relay/index.md index 48e21127..b25e22cb 100755 --- a/doc/chaos/tutorials/relay/index.md +++ b/doc/chaos/tutorials/relay/index.md @@ -6,7 +6,6 @@ start/index merge-composite build-module -transform/index annotated-regions compiler-regions annotate-target diff --git a/doc/chaos/tutorials/transform/index.md b/doc/chaos/tutorials/transform/index.md deleted file mode 100755 index 01850d1c..00000000 --- a/doc/chaos/tutorials/transform/index.md +++ /dev/null @@ -1,11 +0,0 @@ -# TVM 变换 - -```{toctree} -:maxdepth: 2 - -intro -pass -infra -SimplifyInference -custom-pass -``` \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py index 5ee4dcc5..2b377761 100755 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,6 +11,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. import sys from pathlib import Path +from docutils.nodes import literal_block +from pygments.lexers import ClassNotFound, find_lexer_class_by_name +from sphinx.locale import __ +from sphinx.transforms.post_transforms import SphinxPostTransform ROOT = Path(__file__).resolve().parents[1] print(ROOT) @@ -75,9 +79,11 @@ "mystnb.unknown_mime_type", # 禁用 application/vnd.plotly.v1+json and application/vnd.bokehjs_load.v0+json 警告 "myst.xref_missing", # 禁用 myst 警告 "autoapi.python_import_resolution", "autoapi.not_readable" # 禁用 autoapi 警告 + "sphinx_automodapi.automodapi", + "autosectionlabel.*", "autosummary", "intersphinx.external", + "autodoc", "autodoc.import_object" ] - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -328,3 +334,40 @@ autoapi_keep_files = False # 要开始自己编写 API 文档，你可以让 AutoAPI 保留其生成的文件作为基础 autoapi_root = "api" autoapi_member_order = "groupwise" + +class LexerValidation(SphinxPostTransform): + """参考：https://github.com/sphinx-doc/sphinx/issues/11442""" + default_priority = 500 + builders = ('dummy',) + + def __init__(self, *args, **kwargs): + from sphinx.highlighting import lexers, lexer_classes, logger + + super().__init__(*args, **kwargs) + self.lexer_names = lexers.keys() | lexer_classes.keys() + self.logger = logger + + def run(self, **kwargs): + for node in self.document.findall(literal_block): + lang = node.get('language', 'default') + self.logger.info(f"lang, node: {lang, node}") + self.validate_lexer(lang, node) + + def validate_lexer(self, lang, location): + if lang in {'py', 'py3', 'python3', 'default', 'pycon3'}: + lang = 'python' + + if lang in {'C++14', 'C++17', 'C++', 'C++20', 'C++11'}: + lang = 'cpp' + + if lang in self.lexer_names: + return + + try: + lexer = find_lexer_class_by_name(lang) + except ClassNotFound: + self.logger.warning(__('Pygments lexer name %r is not known'), lang, + location=location) + +def setup(app): + app.add_post_transform(LexerValidation) diff --git a/doc/read/ffi/relay-expr.ipynb b/doc/read/ffi/relay-expr.ipynb index 6451dcbc..7cb294f9 100644 --- a/doc/read/ffi/relay-expr.ipynb +++ b/doc/read/ffi/relay-expr.ipynb @@ -16,12 +16,12 @@ "1. `TempExprNode`类：\n", " - 这个类代表临时表达式的基本类型。临时表达式是在重写过程中有用的特定表达式，例如布局或类型转换。\n", " - 它有虚析构函数，这意味着它可以被正确地析构，即使通过基类指针删除派生类对象。\n", - " - 它有纯虚函数 `Realize()`，这个函数的目的是将表达式转换为非临时的表达式。具体实现由子类提供。\n", + " - 它有纯虚函数 `Realize()`，这个函数的目的是将表达式转换为普通的（非临时的）表达式。具体实现由子类提供。\n", " - 它还定义了一些静态常量，包括一个字符串类型的键值 `_type_key`，以及三个布尔类型的值 `_type_has_method_sequal_reduce`、`_type_has_method_shash_reduce` 和 `_type_child_slots`。这些值可能用于在内部跟踪或处理该类型的对象。\n", " - 最后，`TVM_DECLARE_BASE_OBJECT_INFO` 宏用于声明 `TempExprNode` 的一些基本信息，如名称、父类等。\n", "\n", "2. `TempExpr`类：\n", - " - 使用 `TVM_DEFINE_OBJECT_REF_METHODS` 宏来定义一些方法引用。这意味着 `TempExpr` 对象可以像 `RelayExpr` 对象一样使用这些方法。\n", + " - 使用 `TVM_DEFINE_OBJECT_REF_METHODS` 宏来定义一些方法引用。这意味着 `TempExpr` 对象可以像 `RelayExpr` 对象一样使用这些方法。这些方法允许我们在 `RelayExpr` 和 `TempExprNode` 之间进行转换。\n", "\n", "总的来说，这段代码定义了 TVM 中的两种表达式类型：`TempExprNode` 和 `TempExpr`。`TempExpr` 是 `TempExprNode` 的具体实现，可以被用来创建临时表达式。" ] diff --git a/doc/read/index.md b/doc/read/index.md index 64ed288a..4fe847c0 100644 --- a/doc/read/index.md +++ b/doc/read/index.md @@ -24,6 +24,7 @@ name-supply memory-passes uma/index relay/index +qnn/index testing/index codegen/index contrib/index diff --git a/doc/read/qnn/FTVMQnnCanonicalize.ipynb b/doc/read/qnn/FTVMQnnCanonicalize.ipynb new file mode 100644 index 00000000..a90f527d --- /dev/null +++ b/doc/read/qnn/FTVMQnnCanonicalize.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FTVMQnnCanonicalize\n", + "\n", + "源码：`tvm/include/tvm/relay/qnn/transform.h`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```cpp\n", + "/*!\n", + " * \\brief Legalizes a QNN expr. Contains specifically two types of Legalizations. First,\n", + " * converts/Lowers an expression containing QNN ops to an expression containing only core Relay ops.\n", + " * Each QNN op is lowered to a sequence of exisiting Relay ops. This is a target-independent pass.\n", + " * One can register the lowering/transformation function for this op using FTVMQnnCanonicalize\n", + " * attr_name for FTVMLegalize op attribute. Second, as opposed to Relay Legalize, this one legalizes\n", + " * only QNN ops. One can register a transformation/legalization function for an op by using the\n", + " * FTVMQnnLegalize attr_name for FTVMLegalize op attribute. The isolation of QNN and Relay Legalize\n", + " * gives us separation of concerns, leading to a better software practice. The legalization can be\n", + " * configured to happen per target.\n", + " *\n", + " * \\return The pass.\n", + " */\n", + "TVM_DLL Pass Legalize();\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码是一个名为 `Legalize` 的函数，它的作用是将 QNN 表达式合法化。具体来说，它包含两种类型的合法化：\n", + "\n", + "1. 将包含 QNN 算子的表达式转换为仅包含核心 Relay 算子的表达式。每个 QNN 算子都会被转换为一系列现有的 Relay 算子。这是一个与目标 target 无关的传递。可以使用 `FTVMQnnCanonicalize` 属性名称为 `FTVMLegalize` 算子属性注册 transformation/legalization 函数。\n", + "\n", + "2. 与 Relay Legalize 不同，这个函数只对 QNN 算子进行合法化。可以通过使用 `FTVMQnnLegalize` 属性名称为 `FTVMLegalize` 算子属性注册一个算子的 transformation/legalization 函数。QNN 和 Relay Legalize 的隔离使我们能够更好地分离关注点，从而得到更好的软件实践。合法化可以针对每个目标（target）进行配置。\n", + "\n", + "函数返回 Pass 对象。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## {func}`~tvm.relay.qnn.transform.CanonicalizeOps`\n", + "\n", + "源码：`tvm/python/tvm/relay/qnn/transform.py`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import testing" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;31mSignature:\u001b[0m \u001b[0mCanonicalizeOps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m\n", + "Converts/Lowers an expression containing QNN ops to an expression containing only core\n", + "(non-Dialect) Relay ops. Each QNN op is lowered to a sequence of existing Relay ops. This is a\n", + "target-independent pass. One can register the lowering/transformation function for this op using\n", + "FTVMQnnCanonicalize attr_name for FTVMLegalize op attribute. An example of this transformation\n", + "is below\n", + "\n", + "Examples\n", + "________\n", + "\n", + ".. code-block:: python\n", + "\n", + " # Original expression\n", + " qnn_expr = relay.qnn.op.requantize(y,\n", + " input_scale=1,\n", + " input_zero_point=0,\n", + " output_scale=1,\n", + " output_zero_point=0,\n", + " out_dtype='int8')\n", + "\n", + " # We want to utilize all the existing Relay infrastructure. So, instead of supporting this\n", + " # QNN requantize op, we convert it into a sequence of existing Relay operators.\n", + " mod = tvm.IRModule.from_expr(qnn_expr)\n", + " mod = relay.qnn.transform.CanonicalizeOps()(mod)\n", + " relay_expr = mod['main']\n", + " print(relay_expr)\n", + "\n", + " def @main(%quantized_data: Tensor[(200), int32]) -> Tensor[(200), int8] {\n", + " %0 = cast(%quantized_data, dtype=\"int64\") /* ty=Tensor[(200), int64] */;\n", + " %1 = multiply(%0, 2 /* ty=int64 */) /* ty=Tensor[(200), int64] */;\n", + " %2 = multiply(%1, 1073741824 /* ty=int64 */) /* ty=Tensor[(200), int64] */;\n", + " %3 = add(%2, 1073741824 /* ty=int64 */) /* ty=Tensor[(200), int64] */;\n", + " %4 = right_shift(%3, 31 /* ty=int64 */) /* ty=Tensor[(200), int64] */;\n", + " %5 = add(0 /* ty=int64 */, %4) /* ty=Tensor[(200), int64] */;\n", + " %6 = clip(%5, a_min=-128f, a_max=127f) /* ty=Tensor[(200), int64] */;\n", + " cast(%6, dtype=\"int8\") /* ty=Tensor[(200), int8] */\n", + " }\n", + "\n", + "Returns\n", + "-------\n", + "ret : tvm.transform.Pass\n", + " The registered pass that canonicalizes QNN ops to Relay ops.\n", + "\u001b[0;31mFile:\u001b[0m /media/pc/data/lxw/ai/tvm/python/tvm/relay/qnn/transform.py\n", + "\u001b[0;31mType:\u001b[0m function" + ] + } + ], + "source": [ + "from tvm.relay.qnn.transform import CanonicalizeOps\n", + "\n", + "CanonicalizeOps?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为 `CanonicalizeOps` 的函数，它的作用是将包含 QNN 算子的表达式转换为仅包含核心（非 Dialect）Relay 算子的表达式。每个 QNN 算子都会被转换为一系列现有的 Relay 算子。这是一个与目标无关的传递。\n", + "\n", + "函数返回 {class}`tvm.transform.Pass` 对象，该对象将 QNN 算子规范化为 Relay 算子。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "可以使用 `FTVMQnnCanonicalize` 属性名称为 `FTVMLegalize` 算子属性注册 lowering/transformation 函数。" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;31mSignature:\u001b[0m \u001b[0mregister_qnn_canonicalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlegal_op\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m\n", + "Register canonicalization function for a QNN op.\n", + "\n", + "This transforms QNN ops to mainline Relay components.\n", + "\n", + "Parameters\n", + "----------\n", + "op_name : str\n", + " The name of the operator\n", + "\n", + "legal_op: function (Attrs, List[Expr], List[relay.Type]) -> Expr\n", + " The function for transforming an expr to another expr.\n", + "\n", + "level : int\n", + " The priority level\n", + "\u001b[0;31mFile:\u001b[0m /media/pc/data/lxw/ai/tvm/python/tvm/relay/qnn/op/op.py\n", + "\u001b[0;31mType:\u001b[0m function" + ] + } + ], + "source": [ + "from tvm.relay.qnn.op import register_qnn_canonicalize\n", + "\n", + "register_qnn_canonicalize?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/qnn/FTVMQnnLegalize.ipynb b/doc/read/qnn/FTVMQnnLegalize.ipynb new file mode 100644 index 00000000..36a258e8 --- /dev/null +++ b/doc/read/qnn/FTVMQnnLegalize.ipynb @@ -0,0 +1,187 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## {func}`~tvm.relay.qnn.transform.FTVMQnnLegalize`\n", + "\n", + "源码：`tvm/python/tvm/relay/qnn/transform.py` & `tvm/src/relay/qnn/pass/legalize.cc`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import testing" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;31mSignature:\u001b[0m \u001b[0mregister_qnn_legalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlegal_op\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m\n", + "Register legal transformation function for a QNN op.\n", + "\n", + "This helps QNN match hardware intrinsics better and is run before\n", + "canonicalization.\n", + "\n", + "Parameters\n", + "----------\n", + "op_name : str\n", + " The name of the operator\n", + "\n", + "legal_op: function (attrs: Attrs, inputs: List[Expr]) -> new_expr: Expr\n", + " The function for transforming an expr to another expr.\n", + "\n", + "level : int\n", + " The priority level\n", + "\u001b[0;31mFile:\u001b[0m /media/pc/data/lxw/ai/tvm/python/tvm/relay/qnn/op/op.py\n", + "\u001b[0;31mType:\u001b[0m function" + ] + } + ], + "source": [ + "from tvm.relay.qnn.op import register_qnn_legalize\n", + "\n", + "register_qnn_legalize?" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;31mSignature:\u001b[0m \u001b[0mLegalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m\n", + "Legalizes QNN ops. As opposed to Relay Legalize, this one legalizes only QNN ops. One can\n", + "register a transformation/legalization function for an op by using the FTVMQnnLegalize attr_name\n", + "for FTVMLegalize op attribute. The isolation of QNN and Relay Legalize gives us separation of\n", + "concerns, leading to a better software practice. The legalization can be configured to happen\n", + "per target. An example of this type of legalization is shown below.\n", + "\n", + "Examples\n", + "________\n", + "\n", + "Suppose the original graph is as follows\n", + "\n", + " data(u8) weight(u8)\n", + " | |\n", + " | |\n", + " qnn.conv2d (int32)\n", + " |\n", + " |\n", + " nn.relu (int32)\n", + "\n", + "Now, we know that Intel Cascade Lake has VNNI instructions to speedup convolution. However, it\n", + "only works on u8 x i8 inputs. So, here, we can use QNN Legalize to transform the above graph as\n", + "follows\n", + "\n", + " data(u8) weight(u8)\n", + " | |\n", + " | |\n", + " | requantize(i8)\n", + " | |\n", + " | |\n", + " qnn.conv2d (int32)\n", + " |\n", + " |\n", + " nn.relu (int32)\n", + "\n", + "In this legalization, since we have isolated legalization for QNN ops, it will only trigger the\n", + "transformation for qnn.conv2d (and not nn.relu). This pass can be followed by CanonicalizeOps to\n", + "further lower the qnn.requantize and qnn.conv2d into an expr containing only Relay ops.\n", + "\n", + "Returns\n", + "-------\n", + "ret : tvm.transform.Pass\n", + " The registered pass that legalizes QNN ops.\n", + "\u001b[0;31mFile:\u001b[0m /media/pc/data/lxw/ai/tvm/python/tvm/relay/qnn/transform.py\n", + "\u001b[0;31mType:\u001b[0m function" + ] + } + ], + "source": [ + "from tvm.relay.qnn.transform import Legalize\n", + "\n", + "Legalize?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为 `Legalize` 的函数，它的作用是将 QNN 算子合法化。与 Relay Legalize 不同，这个函数只对 QNN 算子进行合法化。可以通过使用 `FTVMQnnLegalize` 属性名称为 `FTVMLegalize` 算子属性注册一个算子的 transformation/legalization 函数。QNN 和 Relay Legalize 的隔离使我们能够更好地分离关注点，从而得到更好的软件实践。合法化可以针对每个目标进行配置。\n", + "\n", + "假设原始图如下：\n", + "\n", + "```\n", + " data(u8) weight(u8)\n", + " | |\n", + " | |\n", + " qnn.conv2d (int32)\n", + " |\n", + " |\n", + " nn.relu (int32)\n", + "```\n", + "\n", + "现在，我们知道 Intel Cascade Lake 有 VNNI 指令来加速卷积。然而，它只适用于 u8 x i8 输入。因此，在这里，我们可以使用 QNN Legalize 将上述图转换为以下形式：\n", + "\n", + "```\n", + " data(u8) weight(u8)\n", + " | |\n", + " | |\n", + " | requantize(i8)\n", + " | |\n", + " | |\n", + " qnn.conv2d (int32)\n", + " |\n", + " |\n", + " nn.relu (int32)\n", + "```\n", + "\n", + "在这个合法化中，由于我们已经为 QNN 算子进行了隔离合法化，它只会触发 `qnn.conv2d` （而不是 `nn.relu`）的转换。此传递可以跟随 `CanonicalizeOps` 进一步将 `qnn.requantize` 和 `qnn.conv2d` 降低为仅包含 Relay 算子的表达式。\n", + "\n", + "函数返回 `tvm.transform.Pass` 对象，该对象将 QNN 算子合法化。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/qnn/index.md b/doc/read/qnn/index.md new file mode 100644 index 00000000..7263a440 --- /dev/null +++ b/doc/read/qnn/index.md @@ -0,0 +1,7 @@ +# QNN + +```{toctree} +FTVMQnnCanonicalize +FTVMQnnLegalize +softmax +``` diff --git a/doc/read/qnn/softmax.ipynb b/doc/read/qnn/softmax.ipynb new file mode 100644 index 00000000..0a4b8823 --- /dev/null +++ b/doc/read/qnn/softmax.ipynb @@ -0,0 +1,313 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# {func}`tvm.relay.qnn.op.softmax`\n", + "\n", + "源码：``tvm/src/relay/qnn/op/softmax.cc``" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import testing" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from tvm import relay" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;31mSignature:\u001b[0m\n", + "\u001b[0mrelay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mscale\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mzero_point\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0moutput_scale\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0moutput_zero_point\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m \n", + "\u001b[0;31mSource:\u001b[0m \n", + "\u001b[0;32mdef\u001b[0m \u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzero_point\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_scale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_zero_point\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_make\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzero_point\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_scale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_zero_point\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mFile:\u001b[0m /media/pc/data/lxw/ai/tvm/python/tvm/relay/qnn/op/qnn.py\n", + "\u001b[0;31mType:\u001b[0m function" + ] + } + ], + "source": [ + "relay.qnn.op.softmax??" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Not equal to tolerance rtol=1e-07, atol=1\n", + "\n", + "Mismatched elements: 5 / 50 (10%)\n", + "Max absolute difference: 5\n", + "Max relative difference: 0.33333333\n", + " x: array([[-128, -128, -128, -128, -128, -128, -128, -128, -128, 126],\n", + " [-128, -128, -128, -128, -128, -128, -128, -124, -96, 88],\n", + " [-128, -128, -128, -128, -128, -128, -128, -128, -120, 118],...\n", + " y: array([[-128, -128, -128, -128, -128, -128, -128, -128, -128, 127],\n", + " [-128, -128, -128, -128, -128, -128, -128, -123, -98, 93],\n", + " [-128, -128, -128, -128, -128, -128, -128, -128, -120, 120],...\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import tvm\n", + "from tvm import relay\n", + "\n", + "is_sorted = lambda a: np.all(a[:-1] <= a[1:])\n", + "\n", + "shape = [5, 10]\n", + "scale = 0.2\n", + "x_ = relay.var(\"x\", shape=shape, dtype=\"int8\")\n", + "x = relay.qnn.op.dequantize(x_, relay.const(scale), relay.const(0))\n", + "op = relay.op.nn.softmax(x, axis=1)\n", + "op = relay.qnn.op.quantize(\n", + " op, relay.const(1.0 / 256.0), relay.const(-128), out_dtype=\"int8\"\n", + ")\n", + "\n", + "x_np = np.random.randint(-128, 127, size=shape, dtype=\"int8\")\n", + "x_np = np.sort(x_np)\n", + "args = [x_np]\n", + "\n", + "mod = tvm.IRModule.from_expr(op)\n", + "mod = tvm.relay.transform.InferType()(mod)\n", + "mod_int = tvm.relay.transform.FakeQuantizationToInteger(\n", + " hard_fail=True, optional_qnn_ops=[\"nn.softmax\"]\n", + ")(mod)\n", + "assert not tvm.ir.structural_equal(mod, mod_int)\n", + "result = (\n", + " relay.create_executor(\"vm\", mod=mod, device=tvm.cpu(), target=\"llvm\")\n", + " .evaluate()(*args)\n", + " .numpy()\n", + ")\n", + "result_int = (\n", + " relay.create_executor(\"vm\", mod=mod_int, device=tvm.cpu(), target=\"llvm\")\n", + " .evaluate()(*args)\n", + " .numpy()\n", + ")\n", + "\n", + "# Check at least the softmax output is in ascending order,\n", + "# since it is difficult to use allclose due to not-so-good accuracy.\n", + "for qdq, qop in zip(result, result_int):\n", + " assert is_sorted(qdq)\n", + " assert is_sorted(qop)\n", + "\n", + "try:\n", + " np.testing.assert_allclose(result_int, result, atol=1)\n", + "except AssertionError as e:\n", + " # To see the difference\n", + " print(e)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
def @main(%x: Tensor[(5, 10), int8] /* ty=Tensor[(5, 10), int8] */) -> Tensor[(5, 10), int8] {\n",
+       "  qnn.softmax(%x, 0.2f /* ty=float32 */, 0 /* ty=int32 */, 0.00390625f /* ty=float32 */, -128 /* ty=int32 */, axis=1) /* ty=Tensor[(5, 10), int8] */\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "mod_int.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
def @main(%x: Tensor[(5, 10), int8] /* ty=Tensor[(5, 10), int8] */) -> Tensor[(5, 10), int8] {\n",
+       "  %0 = cast(%x, dtype="int32") /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %1 = subtract(%0, 0 /* ty=int32 */) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %2 = max(%1, axis=[1], keepdims=True) /* ty=Tensor[(5, 1), int32] */;\n",
+       "  %3 = subtract(%1, %2) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %4 = right_shift(%3, 1 /* ty=int32 */) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %5 = add(%3, %4) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %6 = right_shift(%3, 4 /* ty=int32 */) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %7 = clip(5f /* ty=float32 */, a_min=-2.14748e+09f, a_max=2.14748e+09f) /* ty=float32 */;\n",
+       "  %8 = cast(%7, dtype="int32") /* ty=int32 */;\n",
+       "  %9 = subtract(%5, %6) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %10 = negative(%8) /* ty=int32 */;\n",
+       "  %11 = divide(%9, %10) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %12 = clip(%11, a_min=0f, a_max=20f) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %13 = negative(%8) /* ty=int32 */;\n",
+       "  %14 = multiply(%12, %13) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %15 = subtract(%9, %14) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %16 = right_shift(%15, 1 /* ty=int32 */) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %17 = max(%12, axis=[1], keepdims=True) /* ty=Tensor[(5, 1), int32] */;\n",
+       "  %18 = add(%16, %8) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %19 = subtract(%17, %12) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %20 = left_shift(%18, %19) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %21 = sum(%20, axis=[1], keepdims=True) /* ty=Tensor[(5, 1), int32] */;\n",
+       "  %22 = divide(1073741824 /* ty=int32 */, %21) /* ty=Tensor[(5, 1), int32] */;\n",
+       "  %23 = multiply(%22, %20) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %24 = right_shift(%23, 23 /* ty=int32 */) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %25 = cast(%24, dtype="int32") /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %26 = cast(-128 /* ty=int32 */, dtype="int32") /* ty=int32 */;\n",
+       "  %27 = fixed_point_multiply(%25, multiplier=1073741824, shift=2) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %28 = add(%26, %27) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  %29 = clip(%28, a_min=-128f, a_max=127f) /* ty=Tensor[(5, 10), int32] */;\n",
+       "  cast(%29, dtype="int8") /* ty=Tensor[(5, 10), int8] */\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "with tvm.target.Target(\"llvm\"):\n", + " with tvm.transform.PassContext(opt_level=3):\n", + " run_mod = relay.qnn.transform.Legalize()(mod_int)\n", + " run_mod = relay.qnn.transform.CanonicalizeOps()(run_mod)\n", + "run_mod.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```c++\n", + "/*\n", + " * \\brief Canonicalizes the QNN softmax op.\n", + " * \\param attrs The Softmax attrs.\n", + " * \\param new_args The new mutated args to the call node.\n", + " * \\param arg_types The types of input and output.\n", + " * \\return The sequence of Relay ops for softmax op.\n", + " * \\note This op is highly experimental and sometimes lacks accuracy.\n", + " * Be aware that the input scale must be in the range of 0 to 1.\n", + " */\n", + "Expr QnnSoftmaxCanonicalize(const Attrs& attrs, const Array& new_args,\n", + " const Array& arg_types) {\n", + " // Expected: input, scale, zero_point, output_scale, output_zero_point\n", + " ICHECK_EQ(new_args.size(), 5);\n", + "\n", + " const auto const_i32 = [&](int32_t val) { return MakeConstantScalar(DataType::Int(32), val); };\n", + " const auto const_f32 = [&](float val) { return MakeConstantScalar(DataType::Float(32), val); };\n", + "\n", + " const auto const_input_scale = new_args[1].as();\n", + " ICHECK(const_input_scale) << \"Input scale should be constant.\";\n", + " ICHECK(const_input_scale->is_scalar()) << \"Input scale should be scalar.\";\n", + " const float input_scale = static_cast(const_input_scale->data->data)[0];\n", + " ICHECK(input_scale <= 1.f) << \"Input scale should be less than or equal to 1.\";\n", + "\n", + " const Expr input_zero_point = new_args[2];\n", + " const Expr output_scale = new_args[3];\n", + " const Expr output_zero_point = new_args[4];\n", + " const int axis = attrs.as()->axis;\n", + "\n", + " // Refer to the Algorithm 1 in https://arxiv.org/pdf/2207.01405.pdf\n", + "\n", + " const Expr quantized_data = Subtract(Cast(new_args[0], DataType::Int(32)), input_zero_point);\n", + "\n", + " const Expr x_0 = ConvertDtype(const_f32(std::round(1.f / input_scale)), DataType::Int(32));\n", + " const Expr max = Max(quantized_data, {axis}, true, false);\n", + " const Expr x = Subtract(quantized_data, max);\n", + "\n", + " const int m = 30;\n", + " const int bits = 8;\n", + " const Expr x_p = Subtract(Add(x, RightShift(x, const_i32(1))), RightShift(x, const_i32(4)));\n", + " const Expr q = Clip(Divide(x_p, Negative(x_0)), 0, 20);\n", + " const Expr max_q = Max(q, {axis}, true, false);\n", + " const Expr r = Subtract(x_p, Multiply(q, Negative(x_0)));\n", + " const Expr x_b = Add(RightShift(r, const_i32(1)), x_0);\n", + " const Expr exps = LeftShift(x_b, Subtract(max_q, q));\n", + " const Expr sums = Sum(exps, {axis}, true, false);\n", + " const Expr output =\n", + " RightShift(Multiply(Divide(const_i32(1 << m), sums), exps), const_i32(m - (bits - 1)));\n", + " const Expr requantized = Requantize(output, arg_types[0].as()->shape,\n", + " const_f32(1.f / (1 << (bits - 1))), const_i32(0),\n", + " output_scale, output_zero_point, DataType::Int(bits), 0);\n", + "\n", + " return requantized;\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码是一个用于规范化 QNN softmax 操作的函数。它接受三个参数：`attrs`（Softmax 属性）、`new_args`（新的调用节点参数）和 `arg_types`（输入和输出的类型）。该函数返回 Relay 算子序列，用于执行 softmax 运算。\n", + "\n", + "该函数首先检查输入参数的数量是否正确，然后从 `new_args` 中提取出输入、缩放因子、零点、输出缩放因子和输出零点等参数。接着，它使用这些参数计算出量化数据，并按照[算法 1](https://arxiv.org/pdf/2207.01405.pdf) 进行计算。最后，它将结果重新量化并返回。\n", + "\n", + "需要注意的是，这个函数是高度实验性的，有时可能缺乏准确性。此外，输入缩放因子必须在 0 到 1 之间。" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/qnn/testing.py b/doc/read/qnn/testing.py new file mode 100644 index 00000000..4be69ebe --- /dev/null +++ b/doc/read/qnn/testing.py @@ -0,0 +1,60 @@ +import sys +from pathlib import Path +ROOT = Path(".").resolve().parents[2] +sys.path.extend([f"{ROOT}/tests", f"{ROOT}/src"]) +# # from tools.tag_span import _create_span, _set_span, _verify_structural_equal_with_span +from tools.torch_utils import verify_model +import tvm +from tvm.contrib.relay_viz import RelayVisualizer +from tvm.contrib.relay_viz.dot import DotPlotter +from graphviz import Digraph +from IPython.display import display_svg + + +class Visualizer(RelayVisualizer): + def graph(self, graph_name): + return self._plotter._name_to_graph[graph_name] + + def display(self, graph_name): + graph = self.graph(graph_name) + return graph.digraph + + def display_all(self, format="svg", + filename=None, + directory="images"): + root_graph = Digraph(format=format, + filename=filename, + directory=directory) + for graph in self._plotter._name_to_graph.values(): + root_graph.subgraph(graph.digraph) + return root_graph + +# VizNode is passed to the callback. +# We want to color NCHW conv2d nodes. Also give Var a different shape. +def get_node_attr(node): + if "nn.conv2d" in node.type_name and "NCHW" in node.detail: + return { + "fillcolor": "green", + "style": "filled", + "shape": "box", + } + if "Var" in node.type_name: + return {"shape": "ellipse"} + return {"shape": "box"} + +def viz_expr(expr, func_name="main"): + graph_attr = {"color": "red"} + node_attr = {"color": "blue"} + edge_attr = {"color": "black"} + + # 添加颜色 + dot_plotter = DotPlotter( + graph_attr=graph_attr, + node_attr=node_attr, + edge_attr=edge_attr, + get_node_attr=get_node_attr + ) + mod = tvm.IRModule.from_expr(expr) + viz = Visualizer(mod, plotter=dot_plotter) + graph = viz.display(func_name) + display_svg(graph) diff --git a/doc/read/relay/quant/SimulatedQuantize.ipynb b/doc/read/relay/quant/SimulatedQuantize.ipynb new file mode 100644 index 00000000..5687a1e0 --- /dev/null +++ b/doc/read/relay/quant/SimulatedQuantize.ipynb @@ -0,0 +1,116 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SimulatedQuantize\n", + "\n", + "源码：`tvm/src/relay/quantize/quantize.cc` 和 `tvm/python/tvm/relay/quantize/_annotate.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%cd ..\n", + "import testing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::::{dropdown}\n", + "```c++\n", + "TVM_REGISTER_NODE_TYPE(SimulatedQuantizeAttrs);\n", + "\n", + "bool SimulatedQuantizeRel(const Array& types, int num_inputs, const Attrs& attrs,\n", + " const TypeReporter& reporter) {\n", + " ICHECK_EQ(types.size(), 5);\n", + " const auto param = attrs.as();\n", + " ICHECK(param != nullptr);\n", + "\n", + " const auto* data = types[0].as();\n", + "\n", + " if (data == nullptr) {\n", + " return false;\n", + " }\n", + "\n", + " ICHECK_NE(data->shape.size(), 0) << \"Input shape cannot be empty\";\n", + "\n", + " reporter->Assign(types[1], TensorType({}, DataType::Float(32))); // dom_scale\n", + " reporter->Assign(types[2], TensorType({}, DataType::Float(32))); // clip_min\n", + " reporter->Assign(types[3], TensorType({}, DataType::Float(32))); // clip_max\n", + " reporter->Assign(types[4], types[0]); // output\n", + " return true;\n", + "}\n", + "```\n", + "::::" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了 `SimulatedQuantizeRel` 函数，它的作用是检查输入的类型是否符合预期。具体来说，它首先检查输入的类型数量是否为 5，然后从属性中获取 `SimulatedQuantizeAttrs` 类型的参数。接着，它检查第一个类型是否为 `TensorTypeNode` 类型，如果不是则返回 `false`。最后，它将输出的类型分别设置为 `dom_scale`、`clip_min`、`clip_max` 和输入数据的类型。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::::{dropdown}\n", + "```c++\n", + "RELAY_REGISTER_OP(\"relay.op.annotation.simulated_quantize\")\n", + " .describe(R\"code(simulated quantize op)code\" TVM_ADD_FILELINE)\n", + " .set_num_inputs(4)\n", + " .add_argument(\"data\", \"Tensor\", \"The input data.\")\n", + " .add_argument(\"dom_scale\", \"Tensor\", \"The domain scale of input data. It should be a scalar\")\n", + " .add_argument(\"clip_min\", \"Tensor\", \"lower bound. It should be a scalar\")\n", + " .add_argument(\"clip_max\", \"Tensor\", \"upper bound. It should be a scalar\")\n", + " .set_attrs_type()\n", + " .set_support_level(11)\n", + " .add_type_rel(\"SimulatedQuantize\", SimulatedQuantizeRel);\n", + "\n", + "TVM_REGISTER_GLOBAL(\"relay._quantize.simulated_quantize\")\n", + " .set_body_typed([](Expr data, Expr dom_scale, Expr clip_min, Expr clip_max, int kind, bool sign,\n", + " String rounding) {\n", + " auto attrs = make_object();\n", + " attrs->kind = kind;\n", + " attrs->sign = sign;\n", + " attrs->rounding = rounding;\n", + " static const Op& op = Op::Get(\"relay.op.annotation.simulated_quantize\");\n", + " return Call(op, {data, dom_scale, clip_min, clip_max}, Attrs(attrs), {});\n", + " });\n", + "\n", + "```\n", + "::::" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`RELAY_REGISTER_OP` 宏注册名为 `relay.op.annotation.simulated_quantize` 的算子，该算子有 4 个输入参数：`data`、`dom_scale`、`clip_min` 和 `clip_max`。它还设置了属性类型为 `SimulatedQuantizeAttrs`，并添加了类型关系函数 `SimulatedQuantizeRel`。\n", + "\n", + "`TVM_REGISTER_GLOBAL` 宏注册全局函数 `relay._quantize.simulated_quantize`，该函数接受 6 个参数：`data`、`dom_scale`、`clip_min`、`clip_max`、`kind`、`sign` 和 `rounding`。在这个函数中，首先创建 `SimulatedQuantizeAttrs` 对象，并设置其属性值。然后，调用 `relay.op.annotation.simulated_quantize` 算子，并将结果返回。" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/annotate.ipynb b/doc/read/relay/quant/annotate.ipynb index 16baa4a1..95af442b 100644 --- a/doc/read/relay/quant/annotate.ipynb +++ b/doc/read/relay/quant/annotate.ipynb @@ -11,11 +11,144 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/media/pc/data/lxw/ai/tvm-book/doc/read/relay\n" + ] + } + ], + "source": [ + "%cd ..\n", + "import testing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::::{dropdown}\n", + "```c++\n", + "using namespace relay::transform;\n", + "\n", + "class QAnnotateExpr;\n", + "class QAnnotateExprNode : public TempExprNode {\n", + " public:\n", + " Expr expr;\n", + " QAnnotateKind kind;\n", + "\n", + " void VisitAttrs(tvm::AttrVisitor* v) {\n", + " v->Visit(\"expr\", &expr);\n", + " v->Visit(\"kind\", &kind);\n", + " }\n", + "\n", + " Expr Realize() const final;\n", + "\n", + " static constexpr const char* _type_key = \"relay.QAnnotateExpr\";\n", + " TVM_DECLARE_FINAL_OBJECT_INFO(QAnnotateExprNode, TempExprNode);\n", + "};\n", + "\n", + "class QAnnotateExpr : public TempExpr {\n", + " public:\n", + " /*!\n", + " * \\brief The constructor\n", + " * \\param expr The original relay expression.\n", + " * \\param kind The annotation kind.\n", + " */\n", + " TVM_DLL QAnnotateExpr(Expr expr, QAnnotateKind kind);\n", + "\n", + " TVM_DEFINE_OBJECT_REF_METHODS(QAnnotateExpr, TempExpr, QAnnotateExprNode);\n", + "};\n", + "\n", + "Expr QAnnotateExprNode::Realize() const { return expr; }\n", + "\n", + "QAnnotateExpr::QAnnotateExpr(Expr expr, QAnnotateKind kind) {\n", + " auto rnode = make_object();\n", + " rnode->expr = std::move(expr);\n", + " rnode->kind = kind;\n", + " data_ = std::move(rnode);\n", + "}\n", + "\n", + "TVM_REGISTER_GLOBAL(\"relay._quantize.make_annotate_expr\").set_body_typed([](Expr expr, int kind) {\n", + " return QAnnotateExpr(expr, static_cast(kind));\n", + "});\n", + "```\n", + "::::" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了 `QAnnotateExpr` 类，它继承自 `TempExpr`。这个类主要用于表示带有注解的表达式。其中，`QAnnotateExprNode` 是内部类，用于存储表达式和注解类型。`VisitAttrs` 方法用于访问表达式和注解类型的属性。`Realize` 方法返回原始表达式。\n", + "\n", + "`QAnnotateExpr` 类的构造函数接受一个表达式和一个注解类型作为参数，并将它们存储在 `QAnnotateExprNode` 对象中。`TVM_DEFINE_OBJECT_REF_METHODS` 宏用于定义对象的引用方法。\n", + "\n", + "最后，`TVM_REGISTER_GLOBAL` 宏用于注册全局函数，该函数接受一个表达式和一个整数类型的注解，并返回 `QAnnotateExpr` 对象。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "::::{dropdown}\n", + "```c++\n", + "Pass QuantizeAnnotate() {\n", + " // TODO(tvm-teams): since partition has added cast_hint in different\n", + " // branches, try to remove this in the future.\n", + " std::function fmulti_ref = [](const Expr& e) {\n", + " if (e->IsInstance()) {\n", + " const auto* n = e.as();\n", + " ICHECK(n);\n", + " const PackedFunc* f = runtime::Registry::Get(\"relay.quantize.attach_simulated_quantize\");\n", + " Expr ret = (*f)(n->expr, static_cast(kQInput));\n", + " return static_cast(QAnnotateExpr(ret, kQInput));\n", + " }\n", + " return e;\n", + " };\n", + "\n", + " runtime::TypedPackedFunc pass_func =\n", + " [=](Function f, IRModule m, PassContext pc) {\n", + " auto func = Downcast(ForwardRewrite(f, \"FQAnnotateRewrite\", nullptr, fmulti_ref));\n", + " auto new_params = func->params;\n", + " for (const auto& x : FreeVars(func)) {\n", + " new_params.push_back(x);\n", + " }\n", + " return WithFields(func, new_params);\n", + " };\n", + " return CreateFunctionPass(pass_func, 1, \"QuantizeAnnotate\", {});\n", + "}\n", + "\n", + "TVM_REGISTER_GLOBAL(\"relay._quantize.QuantizeAnnotate\").set_body_typed(QuantizeAnnotate);\n", + "\n", + "TVM_REGISTER_NODE_TYPE(QAnnotateExprNode);\n", + "```\n", + "::::" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了 `QuantizeAnnotate` 函数，它的作用是对输入的函数进行量化注解。具体来说，它首先定义了名为 `fmulti_ref` 的 `lambda` 函数，该函数接受一个表达式作为参数，如果该表达式是 `TempExprNode` 的实例，则对其进行量化注解，否则直接返回原表达式。\n", + "\n", + "接下来，定义了一个名为 `pass_func` 的函数，它接受一个函数、一个 IR 模块和一个 PassContext 作为参数。在这个函数中，首先对输入的函数进行前向重写，然后遍历函数中的自由变量，将它们添加到新的参数列表中。最后，使用新的参数列表创建一个新的函数，并返回。\n", + "\n", + "最后，使用 `CreateFunctionPass` 创建一个函数传递，并将其注册为全局函数 `relay._quantize.QuantizeAnnotate`。同时，还注册了一个名为 `QAnnotateExprNode` 的节点类型。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, "outputs": [], "source": [ "from torch import nn\n", "import torch\n", "\n", + "\n", "class Model(nn.Module):\n", " def __init__(self, *args, **kwargs) -> None:\n", " super().__init__(*args, **kwargs)\n", @@ -34,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -54,7 +187,6 @@ } ], "source": [ - "import set_env\n", "import numpy as np\n", "import tvm\n", "from tvm import relay\n", @@ -75,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -99,7 +231,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 16, 4, 4), float32] */" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -110,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -141,7 +273,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32], float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32) -> Tensor[(1, 16, 4, 4), float32] */" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -159,7 +291,28 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "shape = [5, 10]\n", + "scale = 0.1\n", + "x_ = relay.var(\"x\", shape=shape, dtype=\"int8\")\n", + "x = relay.qnn.op.dequantize(x_, relay.const(scale), relay.const(0))\n", + "op = relay.op.nn.softmax(x, axis=1)\n", + "op = relay.qnn.op.quantize(\n", + " op, relay.const(1.0 / 256.0), relay.const(-128), out_dtype=\"int8\"\n", + ")\n", + "\n", + "x_np = np.random.randint(-128, 127, size=shape, dtype=\"int8\")\n", + "x_np = np.sort(x_np)\n", + "args = [x_np]\n", + "\n", + "mod = tvm.IRModule.from_expr(op)\n", + "mod = tvm.relay.transform.InferType()(mod)\n", + "mod_int = tvm.relay.transform.FakeQuantizationToInteger(\n", + " hard_fail=True, optional_qnn_ops=[\"nn.softmax\"]\n", + ")(mod)\n", + "mod.show()\n", + "mod_int.show()" + ] } ], "metadata": { @@ -178,7 +331,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.3" }, "orig_nbformat": 4 }, diff --git a/doc/read/relay/quant/calibrate.ipynb b/doc/read/relay/quant/calibrate.ipynb new file mode 100644 index 00000000..e060b172 --- /dev/null +++ b/doc/read/relay/quant/calibrate.ipynb @@ -0,0 +1,434 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# TVM 自动量化校准\n", + "\n", + "参考：`tvm/src/relay/quantize/calibrate.cc` 和 `tvm/python/tvm/relay/quantize/_calibrate.py`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/media/pc/data/lxw/ai/tvm-book/doc/read/relay\n" + ] + } + ], + "source": [ + "%cd ..\n", + "import testing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{dropdown}\n", + "```c++\n", + "// KL divergence minimization code is adapted from MXNet.\n", + "// The original one is in incubator-mxnet/src/operator/quantization/calibrate.cc\n", + "static std::vector SmoothDistribution(const std::vector& p,\n", + " const float eps = 0.0001) {\n", + " std::vector is_zeros(p.size());\n", + " std::vector is_nonzeros(p.size());\n", + " {\n", + " auto it = p.begin();\n", + " std::generate(is_zeros.begin(), is_zeros.end(),\n", + " [&it]() { return static_cast(*(it++) == 0.f); });\n", + " }\n", + " {\n", + " auto it = p.begin();\n", + " std::generate(is_nonzeros.begin(), is_nonzeros.end(),\n", + " [&it]() { return static_cast(*(it++) != 0.f); });\n", + " }\n", + " size_t n_zeros = std::accumulate(is_zeros.begin(), is_zeros.end(), 0);\n", + " size_t n_nonzeros = p.size() - n_zeros;\n", + " if (!n_nonzeros) {\n", + " // The discrete probability distribution is malformed. All entries are 0.\n", + " return std::vector();\n", + " }\n", + " float eps1 = eps * static_cast(n_zeros) / static_cast(n_nonzeros);\n", + " if (eps1 >= 1.0) return std::vector();\n", + " auto ret = p;\n", + " for (size_t i = 0; i < p.size(); i++) {\n", + " ret[i] += eps * is_zeros[i] - eps1 * is_nonzeros[i];\n", + " }\n", + " return ret;\n", + "}\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码实现了平滑离散概率分布函数(SmoothDistribution)，用于最小化 KL 散度。该函数接受浮点数向量 `p` 作为输入，并返回平滑后的浮点数向量。\n", + "\n", + "具体实现过程如下：\n", + "1. 首先定义两个大小为 `p.size()` 的整数向量 `is_zeros` 和 `is_nonzeros`，分别用于记录 `p` 中每个元素是否为 `0` 或非 `0`。\n", + "2. 使用 `std::generate` 函数生成 `is_zeros` 和 `is_nonzeros` 向量，其中 `is_zeros[i]` 表示 `p[i]` 是否为 `0`，`is_nonzeros[i]` 表示 `p[i]` 是否非 `0`。\n", + "3. 计算 `p` 中 `0` 的个数 `n_zeros` 和非 0 的个数 `n_nonzeros`。\n", + "4. 如果 `n_nonzeros` 为 `0`，说明离散概率分布格式有误，所有元素都为 `0`，直接返回空向量。\n", + "5. 计算 `eps1`，即 `eps` 乘以 `n_zeros` 除以 `n_nonzeros`。如果 `eps1` 大于等于 `1.0`，也直接返回空向量。\n", + "6. 定义新的向量 `ret`，将 `p`的值复制到 `ret` 中。\n", + "7. 遍历 `p` 中的每个元素，根据 `is_zeros` 和 `is_nonzeros` 的值对 `ret` 进行更新，最终得到平滑后的离散概率分布。\n", + "8. 返回平滑后的向量 `ret`。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{dropdown}\n", + "```c++\n", + "static float ComputeEntropy(float* p, float* q, size_t size) {\n", + " float p_sum = std::accumulate(p, p + size, 0.f);\n", + " float q_sum = std::accumulate(q, q + size, 0.f);\n", + " float ret = 0;\n", + " for (size_t i = 0; i < size; i++) {\n", + " ICHECK(p[i] > 0 && q[i] > 0);\n", + " p[i] /= p_sum;\n", + " q[i] /= q_sum;\n", + " if (p[i] && q[i]) ret += p[i] * std::log(p[i] / q[i]);\n", + " }\n", + " return ret;\n", + "}\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码是一个计算信息熵的函数，输入参数为两个浮点数数组 `p` 和 `q` 以及它们的大小 `size`。函数首先计算 `p` 和 `q` 的元素之和，然后遍历数组，对每个元素进行归一化处理，并计算信息熵。最后返回计算得到的信息熵值。\n", + "\n", + "````{dropdown}\n", + "```c++\n", + "float MinimizeKL(const std::vector& hist, const std::vector& hist_edges, int num_bins,\n", + " int num_quantized_bins) {\n", + " const int zero_bin_idx = num_bins / 2;\n", + " const int num_half_quantized_bins = num_quantized_bins / 2;\n", + " std::vector thresholds(num_bins / 2 + 1 - num_quantized_bins / 2, 0.f);\n", + " std::vector divergence(thresholds.size(), 0.f);\n", + " std::vector quantized_bins(num_quantized_bins, 0);\n", + " for (int i = num_quantized_bins / 2; i < zero_bin_idx + 1; ++i) {\n", + " const int p_bin_idx_start = zero_bin_idx - i;\n", + " const int p_bin_idx_stop = zero_bin_idx + i + 1;\n", + " thresholds[i - num_half_quantized_bins] = hist_edges[p_bin_idx_stop];\n", + "\n", + " std::vector sliced_nd_hist(p_bin_idx_stop - p_bin_idx_start);\n", + " std::vector p(sliced_nd_hist.size());\n", + " p[0] = 0;\n", + " p.back() = 0;\n", + " for (int j = 0; j < num_bins; j++) {\n", + " if (j <= p_bin_idx_start) {\n", + " p[0] += hist[j];\n", + " } else if (j >= p_bin_idx_stop) {\n", + " p.back() += hist[j];\n", + " } else {\n", + " sliced_nd_hist[j - p_bin_idx_start] = hist[j];\n", + " p[j - p_bin_idx_start] = hist[j];\n", + " }\n", + " }\n", + " // calculate how many bins should be merged to generate quantized distribution q\n", + " const auto num_merged_bins = sliced_nd_hist.size() / num_quantized_bins;\n", + " for (int j = 0; j < num_quantized_bins; j++) {\n", + " const int start = j * num_merged_bins;\n", + " const int stop = (j + 1) * num_merged_bins;\n", + " quantized_bins[j] =\n", + " std::accumulate(sliced_nd_hist.begin() + start, sliced_nd_hist.begin() + stop, 0);\n", + " }\n", + " quantized_bins.back() += std::accumulate(\n", + " sliced_nd_hist.begin() + static_cast(num_quantized_bins * num_merged_bins),\n", + " sliced_nd_hist.end(), 0);\n", + " // expand quantized_bins into p.size bins\n", + " std::vector q(sliced_nd_hist.size(), 0);\n", + " for (int j = 0; j < num_quantized_bins; j++) {\n", + " const int start = j * num_merged_bins;\n", + " const int stop = (j == num_quantized_bins - 1) ? q.size() : ((j + 1) * num_merged_bins);\n", + " int norm = std::count_if(sliced_nd_hist.begin() + start, sliced_nd_hist.begin() + stop,\n", + " [](size_t i) { return i != 0; });\n", + " if (norm) {\n", + " for (int k = start; k < stop; k++) {\n", + " if (p[k]) q[k] = quantized_bins[j] / norm;\n", + " }\n", + " }\n", + " }\n", + " p = SmoothDistribution(p);\n", + " q = SmoothDistribution(q);\n", + "\n", + " if (!q.size()) {\n", + " divergence[i - num_half_quantized_bins] = std::numeric_limits::infinity();\n", + " } else {\n", + " divergence[i - num_half_quantized_bins] = ComputeEntropy(p.data(), q.data(), p.size());\n", + " }\n", + " }\n", + " auto min_divergence_idx =\n", + " std::distance(divergence.begin(), std::min_element(divergence.begin(), divergence.end()));\n", + " return thresholds[min_divergence_idx];\n", + "}\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码是一个最小化 KL 散度的函数，输入参数为一个整数向量 hist、一个浮点数向量 hist_edges、两个整数 num_bins 和 num_quantized_bins。函数首先定义了一些变量，包括零分箱索引zero_bin_idx、半量化分箱数num_half_quantized_bins、阈值向量thresholds、发散度向量divergence和量化分箱向量quantized_bins。然后，函数遍历hist_edges中的元素，计算每个元素对应的p和q分布，并计算它们之间的KL散度。最后，函数返回具有最小 KL 散度的阈值。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{dropdown}\n", + "```c++\n", + "class StatsCollector : private ExprMutator {\n", + " public:\n", + " StatsCollector() : simulated_quantize_op_(Op::Get(\"relay.op.annotation.simulated_quantize\")) {}\n", + "\n", + " Expr Collect(const Expr& expr) {\n", + " auto new_e = this->Mutate(expr);\n", + " const FunctionNode* func = new_e.as();\n", + " ICHECK(func) << \"Input shoule be Function\";\n", + " Expr new_body = Tuple(std::move(profile_data_));\n", + " Function ret_func = WithFields(GetRef(func), FreeVars(new_body), new_body);\n", + "\n", + " // We are changing the function's ret_type to an empty type. Unfortunately, Optional() is\n", + " // indistinguishable from NullValue(), so we can't express \"update to nullptr\" in\n", + " // WithFields.\n", + " ret_func.CopyOnWrite()->ret_type = NullValue();\n", + " return std::move(ret_func);\n", + " }\n", + "\n", + " private:\n", + " Array profile_data_;\n", + " const Op& simulated_quantize_op_;\n", + "\n", + " Expr VisitExpr_(const CallNode* call) {\n", + " Expr new_e = ExprMutator::VisitExpr_(call);\n", + " const CallNode* new_call = new_e.as();\n", + " ICHECK(new_call);\n", + " if (new_call->op == simulated_quantize_op_) {\n", + " auto attrs = new_call->attrs.as();\n", + " // rewrite the annotation\n", + " auto new_attrs = make_object();\n", + " const Expr& quantize_input = new_call->args[0]; // expression being quantized\n", + " auto placeholder = MakeConstantScalar(DataType::Float(32), 0.); // unused argument\n", + " Array new_args{quantize_input, placeholder, placeholder, placeholder};\n", + " new_attrs->kind = QAnnotateKind::kQIdentity;\n", + " new_attrs->sign = attrs->sign;\n", + " new_attrs->rounding = attrs->rounding;\n", + " Expr identity_quantize = Call(new_call->op, new_args, Attrs{new_attrs}, {});\n", + "\n", + " // add non-const expressions to profile data\n", + " if (attrs->kind != QAnnotateKind::kQWeight) {\n", + " ICHECK(!quantize_input.as());\n", + " profile_data_.push_back(identity_quantize);\n", + " }\n", + " return identity_quantize;\n", + " } else {\n", + " return new_e;\n", + " }\n", + " }\n", + "};\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了 StatsCollector 类，它继承自 ExprMutator 类。该类的主要作用是收集表达式中的量化信息，并将这些信息存储在 `profile_data_` 数组中。\n", + "\n", + "在Collect函数中，首先调用Mutate函数对输入的表达式进行遍历和修改，然后将其转换为FunctionNode类型，并检查其是否为空。接着，将 `profile_data_` 数组转换为 Tuple 类型，并将其作为新的函数体。最后，将新函数的返回类型设置为 `NullValue()`，表示返回类型为空。\n", + "\n", + "在 `VisitExpr_` 函数中，首先调用 `ExprMutator::VisitExpr_` 函数对 `CallNode` 类型的节点进行处理。如果该节点算子是 `simulated_quantize_op_`，则获取该节点的属性，并创建一个新的 `SimulatedQuantizeAttrs` 对象。接着，将该节点的第一个参数作为量化表达式，创建一个占位符常量，并将它们与新属性一起传递给 `Call` 函数，生成 ``identity_quantize`` 节点。如果该节点的属性 `kind` 不等于 `kQWeight`，则将非 `const` 表达式添加到 `profile_data_` 数组中。最后，返回 `identity_quantize` 节点。否则，直接返回 `new_e` 节点。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{dropdown}\n", + "```c++\n", + "/*\n", + " * \\brief Given an annotated graph, create a profile graph to collect profile data from the\n", + " * calibration dataset.\n", + " *\n", + " * This pass collects simulated_quantize op into a tuple. Simulated_quantize ops are rewritten to\n", + " * identity mode. The tuple is the output of the profile graph. Both input and output of this pass\n", + " * are relay::Function.\n", + " *\n", + " * \\param expr The simulation graph after annotation.\n", + " * \\return The profile graph.\n", + " */\n", + "Expr CreateStatsCollector(const Expr& expr) { return StatsCollector().Collect(expr); }\n", + "\n", + "TVM_REGISTER_GLOBAL(\"relay._quantize.CreateStatsCollector\").set_body_typed(CreateStatsCollector);\n", + "\n", + "TVM_REGISTER_GLOBAL(\"relay._quantize.FindScaleByKLMinimization\")\n", + " .set_body([](TVMArgs args, TVMRetValue* ret) {\n", + " int* hist_ptr = static_cast(static_cast(args[0]));\n", + " float* hist_edges_ptr = static_cast(static_cast(args[1]));\n", + " int num_bins = args[2];\n", + " int num_quantized_bins = args[3];\n", + " std::vector hist(hist_ptr, hist_ptr + num_bins);\n", + " std::vector hist_edges(hist_edges_ptr, hist_edges_ptr + num_bins + 1);\n", + " ret[0] = MinimizeKL(hist, hist_edges, num_bins, num_quantized_bins);\n", + " });\n", + "\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为 `StatsCollector` 的类，它继承自 `ExprMutator` 类。该类的主要作用是收集表达式中的量化信息，并将这些信息存储在 `profile_data_` `数组中。\n", + "\n", + "在Collect函数中，首先调用Mutate函数对输入的表达式进行遍历和修改，然后将其转换为FunctionNode类型，并检查其是否为空。接着，将profile_data_数组转换为Tuple类型，并将其作为新的函数体。最后，将新函数的返回类型设置为 `NullValue()`，表示返回类型为空。\n", + "\n", + "在VisitExpr_函数中，首先调用ExprMutator::VisitExpr_函数对CallNode类型的节点进行处理。如果该节点的操作符是simulated_quantize_op_，则获取该节点的属性，并创建一个新的SimulatedQuantizeAttrs对象。接着，将该节点的第一个参数作为量化表达式，创建一个占位符常量，并将它们与新属性一起传递给Call函数，生成一个identity_quantize节点。如果该节点的属性kind不等于kQWeight，则将非const表达式添加到profile_data_数组中。最后，返回identity_quantize节点。否则，直接返回new_e节点。\n", + "\n", + "此外，还定义了两个全局变量：`CreateStatsCollector` 和 ``FindScaleByKLMinimization``。`CreateStatsCollector` 用于创建统计收集器，而 `FindScaleByKLMinimization` 用于通过KL最小化方法查找scale。" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;31mSignature:\u001b[0m\n", + "\u001b[0m_find_scale_by_kl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mquantized_dtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'int8'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mnum_bins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8001\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mnum_quantized_bins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m255\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mSource:\u001b[0m \n", + "\u001b[0;32mdef\u001b[0m \u001b[0m_find_scale_by_kl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquantized_dtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"int8\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_bins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8001\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_quantized_bins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m255\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;34m\"\"\"Given a tensor, find the optimal threshold for quantizing it.\u001b[0m\n", + "\u001b[0;34m The reference distribution is `q`, and the candidate distribution is `p`.\u001b[0m\n", + "\u001b[0;34m `q` is a truncated version of the original distribution.\u001b[0m\n", + "\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m Ref:\u001b[0m\n", + "\u001b[0;34m http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf\u001b[0m\n", + "\u001b[0;34m \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mmin_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mmax_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mthres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmin_val\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mquantized_dtype\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"uint8\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;31m# We need to move negative bins to positive bins to fit uint8 range.\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mnum_quantized_bins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnum_quantized_bins\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_pointer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctypes_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mptr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_as\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPOINTER\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctypes_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mptr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_void_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mhist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhist_edges\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_bins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mthres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mhist_ptr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_pointer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mhist_edges_ptr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_pointer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist_edges\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_float\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_quantize\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFindScaleByKLMinimization\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mhist_ptr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhist_edges_ptr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_bins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_quantized_bins\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mFile:\u001b[0m /media/pc/data/lxw/ai/tvm/python/tvm/relay/quantize/kl_divergence.py\n", + "\u001b[0;31mType:\u001b[0m function" + ] + } + ], + "source": [ + "from tvm.relay.quantize.kl_divergence import _find_scale_by_kl\n", + "\n", + "_find_scale_by_kl??" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "示例：" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "频数： [ 0 0 0 3 2 22 50 81 165 174 188 148 101 45 12 8 1 0\n", + " 0 0]\n", + "分箱边界： [-5. -4.5 -4. -3.5 -3. -2.5 -2. -1.5 -1. -0.5 0. 0.5 1. 1.5\n", + " 2. 2.5 3. 3.5 4. 4.5 5. ]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "# 生成随机数据\n", + "data = np.random.randn(1000)\n", + "\n", + "# 定义分箱边界\n", + "bin_edges = np.linspace(-5, 5, 21)\n", + "\n", + "# 计算直方图\n", + "hist, bin_edges = np.histogram(data, bins=bin_edges)\n", + "\n", + "print(\"频数：\", hist)\n", + "print(\"分箱边界：\", bin_edges)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/index.md b/doc/read/relay/quant/index.md index c70a65cc..183adcc1 100644 --- a/doc/read/relay/quant/index.md +++ b/doc/read/relay/quant/index.md @@ -8,5 +8,8 @@ prerequisite-optimize QPartitionExpr partition annotate +SimulatedQuantize +calibrate +realize/index partition-conversions ``` diff --git a/doc/read/relay/quant/partition.ipynb b/doc/read/relay/quant/partition.ipynb index 52051b71..f925d7e3 100644 --- a/doc/read/relay/quant/partition.ipynb +++ b/doc/read/relay/quant/partition.ipynb @@ -59,7 +59,25 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/media/pc/data/lxw/ai/tvm-book/doc/read/relay\n" + ] + } + ], + "source": [ + "%cd ..\n", + "import testing" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -116,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -145,7 +163,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 8), float32] */" ] }, - "execution_count": 3, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -163,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -183,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -208,7 +226,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 8), float32] */" ] }, - "execution_count": 5, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -227,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -241,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -271,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -303,7 +321,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -347,7 +365,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -361,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -394,7 +412,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 8), float32] */" ] }, - "execution_count": 11, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -413,7 +431,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -427,7 +445,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -454,7 +472,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 8), float32] */" ] }, - "execution_count": 13, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -473,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -482,7 +500,7 @@ "" ] }, - "execution_count": 14, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -505,7 +523,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -538,7 +556,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 8), float32] */" ] }, - "execution_count": 15, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -557,7 +575,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -600,7 +618,7 @@ "} /* ty=fn (Tensor[(1, 3, 4, 4), float32]) -> Tensor[(1, 8), float32] */" ] }, - "execution_count": 16, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -687,7 +705,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -723,7 +741,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -732,7 +750,7 @@ "text": [ "def @main(%data: Tensor[(1, 3, 4, 4), float32] /* ty=Tensor[(1, 3, 4, 4), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", " %44 = fn (%p08: Tensor[(1, 3, 4, 4), float32] /* ty=Tensor[(1, 3, 4, 4), float32] */, Primitive=1) -> Tensor[(1, 3, 4, 4), int8] {\n", - " %41 = multiply(%p08, 90.6147f /* ty=float32 */) /* ty=Tensor[(1, 3, 4, 4), float32] */;\n", + " %41 = multiply(%p08, 97.7937f /* ty=float32 */) /* ty=Tensor[(1, 3, 4, 4), float32] */;\n", " %42 = round(%41) /* ty=Tensor[(1, 3, 4, 4), float32] */;\n", " %43 = clip(%42, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 3, 4, 4), float32] */;\n", " cast(%43, dtype=\"int8\") /* ty=Tensor[(1, 3, 4, 4), int8] */\n", @@ -742,7 +760,7 @@ " %35 = nn.conv2d(%p07, %p14, padding=[1, 1, 1, 1], channels=16, kernel_size=[3, 3], out_dtype=\"int32\") /* ty=Tensor[(1, 16, 4, 4), int32] */;\n", " %36 = nn.relu(%35) /* ty=Tensor[(1, 16, 4, 4), int32] */;\n", " %37 = cast(%36, dtype=\"int64\") /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", - " %38 = fixed_point_multiply(%37, multiplier=1993440000, shift=-9) /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", + " %38 = fixed_point_multiply(%37, multiplier=1144341760, shift=-8) /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", " %39 = clip(%38, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", " %40 = cast(%39, dtype=\"int32\") /* ty=Tensor[(1, 16, 4, 4), int32] */;\n", " cast(%40, dtype=\"int8\") /* ty=Tensor[(1, 16, 4, 4), int8] */\n", @@ -753,7 +771,7 @@ " %29 = add(%28, %p22) /* ty=Tensor[(1, 16, 4, 4), int32] */;\n", " %30 = nn.relu(%29) /* ty=Tensor[(1, 16, 4, 4), int32] */;\n", " %31 = cast(%30, dtype=\"int64\") /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", - " %32 = fixed_point_multiply(%31, multiplier=1769920128, shift=-9) /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", + " %32 = fixed_point_multiply(%31, multiplier=1562188928, shift=-9) /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", " %33 = clip(%32, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 16, 4, 4), int64] */;\n", " %34 = cast(%33, dtype=\"int32\") /* ty=Tensor[(1, 16, 4, 4), int32] */;\n", " cast(%34, dtype=\"int8\") /* ty=Tensor[(1, 16, 4, 4), int8] */\n", @@ -767,9 +785,9 @@ " %52 = fn (%p04: Tensor[(1, 16, 2, 2), int8] /* ty=Tensor[(1, 16, 2, 2), int8] */, Primitive=1) -> Tensor[(1, 64), int8] {\n", " %20 = reshape(%p04, newshape=[0, -1, 1, 1]) /* ty=Tensor[(1, 64, 1, 1), int8] */;\n", " %21 = cast(%20, dtype=\"float32\") /* ty=Tensor[(1, 64, 1, 1), float32] */;\n", - " %22 = multiply(%21, 0.00368804f /* ty=float32 */) /* ty=Tensor[(1, 64, 1, 1), float32] */;\n", + " %22 = multiply(%21, 0.00336449f /* ty=float32 */) /* ty=Tensor[(1, 64, 1, 1), float32] */;\n", " %23 = squeeze(%22, axis=[2, 3]) /* ty=Tensor[(1, 64), float32] span=aten__flatten_0:0:0 */;\n", - " %24 = multiply(%23, 287.053f /* ty=float32 */) /* ty=Tensor[(1, 64), float32] */;\n", + " %24 = multiply(%23, 298.605f /* ty=float32 */) /* ty=Tensor[(1, 64), float32] */;\n", " %25 = round(%24) /* ty=Tensor[(1, 64), float32] */;\n", " %26 = clip(%25, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 64), float32] */;\n", " cast(%26, dtype=\"int8\") /* ty=Tensor[(1, 64), int8] */\n", @@ -779,7 +797,7 @@ " %14 = nn.dense(%p03, %p12, units=None, out_dtype=\"int32\") /* ty=Tensor[(1, 32), int32] */;\n", " %15 = nn.relu(%14) /* ty=Tensor[(1, 32), int32] */;\n", " %16 = cast(%15, dtype=\"int64\") /* ty=Tensor[(1, 32), int64] */;\n", - " %17 = fixed_point_multiply(%16, multiplier=1294491008, shift=-8) /* ty=Tensor[(1, 32), int64] */;\n", + " %17 = fixed_point_multiply(%16, multiplier=1383273600, shift=-8) /* ty=Tensor[(1, 32), int64] */;\n", " %18 = clip(%17, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 32), int64] */;\n", " %19 = cast(%18, dtype=\"int32\") /* ty=Tensor[(1, 32), int32] */;\n", " cast(%19, dtype=\"int8\") /* ty=Tensor[(1, 32), int8] */\n", @@ -790,7 +808,7 @@ " %8 = add(%7, %p21) /* ty=Tensor[(1, 16), int32] */;\n", " %9 = nn.relu(%8) /* ty=Tensor[(1, 16), int32] */;\n", " %10 = cast(%9, dtype=\"int64\") /* ty=Tensor[(1, 16), int64] */;\n", - " %11 = fixed_point_multiply(%10, multiplier=1425444608, shift=-9) /* ty=Tensor[(1, 16), int64] */;\n", + " %11 = fixed_point_multiply(%10, multiplier=2026565248, shift=-9) /* ty=Tensor[(1, 16), int64] */;\n", " %12 = clip(%11, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 16), int64] */;\n", " %13 = cast(%12, dtype=\"int32\") /* ty=Tensor[(1, 16), int32] */;\n", " cast(%13, dtype=\"int8\") /* ty=Tensor[(1, 16), int8] */\n", @@ -800,7 +818,7 @@ " %1 = nn.dense(%p01, %p1, units=None, out_dtype=\"int32\") /* ty=Tensor[(1, 8), int32] */;\n", " %2 = add(%1, %p2) /* ty=Tensor[(1, 8), int32] */;\n", " %3 = cast(%2, dtype=\"int64\") /* ty=Tensor[(1, 8), int64] */;\n", - " %4 = fixed_point_multiply(%3, multiplier=1315281408, shift=-8) /* ty=Tensor[(1, 8), int64] */;\n", + " %4 = fixed_point_multiply(%3, multiplier=1088137088, shift=-9) /* ty=Tensor[(1, 8), int64] */;\n", " %5 = clip(%4, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8), int64] */;\n", " %6 = cast(%5, dtype=\"int32\") /* ty=Tensor[(1, 8), int32] */;\n", " cast(%6, dtype=\"int8\") /* ty=Tensor[(1, 8), int8] */\n", @@ -808,7 +826,7 @@ " %59 = %58(%57, meta[relay.Constant][6] /* ty=Tensor[(8, 16), int8] */, meta[relay.Constant][7] /* ty=Tensor[(8), int32] */) /* ty=Tensor[(1, 8), int8] */;\n", " %60 = fn (%p0: Tensor[(1, 8), int8] /* ty=Tensor[(1, 8), int8] */, Primitive=1) -> Tensor[(1, 8), float32] {\n", " %0 = cast(%p0, dtype=\"float32\") /* ty=Tensor[(1, 8), float32] */;\n", - " multiply(%0, 0.00124995f /* ty=float32 */) /* ty=Tensor[(1, 8), float32] */\n", + " multiply(%0, 0.00189963f /* ty=float32 */) /* ty=Tensor[(1, 8), float32] */\n", " } /* ty=fn (Tensor[(1, 8), int8]) -> Tensor[(1, 8), float32] */;\n", " %60(%59) /* ty=Tensor[(1, 8), float32] */\n", "}\n", @@ -845,7 +863,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.3" }, "orig_nbformat": 4 }, diff --git a/doc/read/relay/quant/realize/CastDtypeInputRealize.ipynb b/doc/read/relay/quant/realize/CastDtypeInputRealize.ipynb new file mode 100644 index 00000000..6d2bd0d4 --- /dev/null +++ b/doc/read/relay/quant/realize/CastDtypeInputRealize.ipynb @@ -0,0 +1,52 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CastDtypeInputRealize\n", + "\n", + "参考：`tvm/src/relay/quantize/realize.cc`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```cpp\n", + "/* \\brief for unary operators which requantize its input to dtype_nbit */\n", + "Expr CastDtypeInputRealize(const Call& ref_call, const Array& new_args,\n", + " const ObjectRef& ctx) {\n", + " const QConfig& cfg = QConfig::Current();\n", + " ICHECK_EQ(new_args.size(), 1);\n", + " if (const auto* n = new_args[0].as()) {\n", + " Expr data = Cast(n->data, cfg->dtype_input);\n", + " Expr ret = ForwardOp(ref_call, {data});\n", + " return QRealizeIntExpr(ret, n->dom_scale, cfg->dtype_input);\n", + " }\n", + " ICHECK(!new_args[0]->IsInstance());\n", + " return Expr(nullptr);\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为 `CastDtypeInputRealize` 的函数，它接受三个参数：`ref_call`、`new_args` 和 `ctx`。该函数的作用是将一元算子的输入重新量化为 `dtype_nbit` 类型。\n", + "\n", + "首先，获取当前的 `QConfig` 对象 `cfg`。然后检查 `new_args` 数组的大小是否为 `1`。如果 `new_args[0]` 是 `QRealizeIntExprNode` 类型的实例，那么将 `n->data` 转换为 `cfg->dtype_input` 类型，并将结果存储在 `data` 中。接着，使用 `ForwardOp` 函数将 `ref_call` 和 `{data}` 作为参数传递，并将结果存储在 `ret` 中。最后，返回 `QRealizeIntExpr` 对象，其中包含 `ret`、`n->dom_scale` 和 `cfg->dtype_input`。\n", + "\n", + "如果 `new_args[0]` 不是 `TempExprNode` 类型的实例，那么返回空的 `Expr` 对象。" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/realize/GetFixedPointMultiplierShift.ipynb b/doc/read/relay/quant/realize/GetFixedPointMultiplierShift.ipynb new file mode 100644 index 00000000..b5ec1522 --- /dev/null +++ b/doc/read/relay/quant/realize/GetFixedPointMultiplierShift.ipynb @@ -0,0 +1,168 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GetFixedPointMultiplierShift\n", + "\n", + "源码：`tvm/src/relay/qnn/utils.cc`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```c++\n", + "/*\n", + " * \\brief Convert FP32 representation into fixed point representation.\n", + " * \\param double_multplier The input FP32 number.\n", + " * \\return The pair of multiplier and shift for fixed point representation.\n", + " * \\note Converts a floating point number so that it can be represented by\n", + " * integers. The representation is\n", + " * float_number = (significand) * 2^(exponent)\n", + " *\n", + " * The significand is a number between 0.5 and 1. This is represented by\n", + " * an integer number. For example, if it is int32, then the decimal point\n", + " * exists between bit 31 and 30 from LSB (or between first and second bit\n", + " * from the left).\n", + " *\n", + " * Some examples are\n", + " * 0.25 = (0.5) * 2^(-1)\n", + " * 0.125 = (0.5) * 2^(-2)\n", + " *\n", + " * Credit to TFLite reference implementation.\n", + " */\n", + "std::pair GetFixedPointMultiplierShift(double double_multiplier) {\n", + " int32_t significand, exponent;\n", + " if (double_multiplier == 0.) {\n", + " significand = 0;\n", + " exponent = 0;\n", + " return std::make_pair(significand, exponent);\n", + " }\n", + "\n", + " // Get the significand and exponent.\n", + " double significand_d = std::frexp(double_multiplier, &exponent);\n", + "\n", + " // Convert the double significand to int significand, i.e., convert into a\n", + " // integer where the decimal point is between bit 31 and 30. This is done by\n", + " // multiplying the double value with 2^31 and then casting to int.\n", + " significand_d = std::round(significand_d * (1ll << 31));\n", + " auto significand_int64 = static_cast(significand_d);\n", + " ICHECK_LE(significand_int64, (1ll << 31));\n", + " if (significand_int64 == (1ll << 31)) {\n", + " significand_int64 /= 2;\n", + " ++exponent;\n", + " }\n", + " ICHECK_LE(significand_int64, std::numeric_limits::max());\n", + " significand = static_cast(significand_int64);\n", + " return std::make_pair(significand, exponent);\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "函数 `GetFixedPointMultiplierShift`，它接受双精度浮点数 `double_multiplier` 作为参数，返回包含两个整数的 pair 对象。\n", + "\n", + "该函数的作用是将输入的双精度浮点数转换为定点数表示形式，即将其转换为具有固定小数位数的数值。具体来说，它将输入的浮点数分解为尾数和指数两部分，并将尾数转换为整数，使得小数点位于第 31 位和第 30 位之间。然后，将这个整数和指数一起返回。\n", + "\n", + "在函数内部，首先判断输入的浮点数是否为零，如果是，则直接返回零值对。否则，使用 `std::frexp` 函数获取浮点数的尾数和指数。接着，将尾数乘以 $2^{31}$，并四舍五入得到整数。如果这个整数等于 $2^{31}$，则将其除以2，并将指数加1。最后，将整数转换为 `int32_t` 类型，并检查其是否小于等于 `int32_t` 的最大值。如果满足条件，则将其和指数一起返回。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`std::frexp` 是 C++ 标准库中的一个函数，用于将一个浮点数分解为尾数和指数。它的原型如下：\n", + "\n", + "```cpp\n", + "double frexp(double x, int* exp);\n", + "```\n", + "\n", + "参数：\n", + "- `x`：要分解的浮点数。\n", + "- `exp`：指向一个整数的指针，用于存储分解后的指数(xponent)部分。\n", + "\n", + "返回值：\n", + "- 返回分解后的尾数(Mantissa)部分。" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "#include \n", + "#include \n", + "#include \n", + "#include " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "std::frexp(x, &n) => 16.4 = 0.5125 * (1 << 5)" + ] + } + ], + "source": [ + "#include \n", + "#include \n", + "\n", + "double x, y;\n", + "int n;\n", + "x = 16.4;\n", + "y = frexp(x, &n);\n", + "std::cout << \"std::frexp(x, &n) => \" \n", + " << x << \" = \" << y << \" * \"\n", + " << \"(1 << \" << n << \")\";" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "16.400000" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "0.5125 * (1 << 5)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "C++14", + "language": "C++14", + "name": "xcpp14" + }, + "language_info": { + "codemirror_mode": "text/x-c++src", + "file_extension": ".cpp", + "mimetype": "text/x-c++src", + "name": "c++", + "version": "14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/realize/MulAndDiv.ipynb b/doc/read/relay/quant/realize/MulAndDiv.ipynb new file mode 100644 index 00000000..f08934a3 --- /dev/null +++ b/doc/read/relay/quant/realize/MulAndDiv.ipynb @@ -0,0 +1,76 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MulAndDiv\n", + "\n", + "参考：`tvm/src/relay/quantize/realize.cc`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```c++\n", + "/* calculate `data * s1 / s2`, use shift if possible */\n", + "inline Expr MulAndDiv(Expr data, float s1, float s2, DataType dtype,\n", + " const Array& data_shape) {\n", + " const QConfig& cfg = QConfig::Current();\n", + " // here we assume the dtype of data is dtype activation\n", + " if (s1 == s2) return data;\n", + "\n", + " float factor = s1 / s2;\n", + " float shift_factor = std::log2(factor);\n", + " ICHECK_GT(shift_factor, 0);\n", + " if (static_cast(shift_factor) == shift_factor) {\n", + " return LeftShift(data, MakeConstantScalar(dtype, static_cast(shift_factor)));\n", + " } else if (static_cast(factor) == factor) {\n", + " return Multiply(data, MakeConstantScalar(dtype, factor));\n", + " } else {\n", + " if (cfg->rounding == \"UPWARD\") {\n", + " auto [fixed_point_multiplier, shift] = qnn::GetFixedPointMultiplierShift(factor);\n", + " data = relay::FixedPointMultiply(data, fixed_point_multiplier, shift);\n", + " } else {\n", + " data = qnn::FixedPointMultiplyToNearest(data, factor, data_shape);\n", + " }\n", + "\n", + " return Cast(data, dtype);\n", + " }\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为`MulAndDiv`的内联函数，用于计算 `data * s1 / s2`。如果可能的话，它会使用位移运算来优化计算过程。\n", + "\n", + "函数接收5个参数：\n", + "- `data`：需要进行计算的数据；\n", + "- `s1` 和 `s2`：两个浮点数，用于计算 `data * s1 / s2`；\n", + "- `dtype`：数据类型；\n", + "- `data_shape`：数据的形状。\n", + "\n", + "函数首先获取当前的量化配置（`QConfig::Current()`），然后判断 `s1` 和 `s2` 是否相等，如果相等则直接返回 `data`。\n", + "\n", + "接下来，计算 `factor = s1 / s2`，并计算 `shift_factor = std::log2(factor)`。如果 `shift_factor` 大于 0 且为整数，则对 `data` 进行左移运算。如果 `factor` 为整数，则对 `data` 进行乘法运算。否则，根据量化配置中的舍入方式（`cfg->rounding`）进行定点乘法运算，并将结果转换为指定的数据类型。" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/realize/UnifyDTypeScale.ipynb b/doc/read/relay/quant/realize/UnifyDTypeScale.ipynb new file mode 100644 index 00000000..ef71319e --- /dev/null +++ b/doc/read/relay/quant/realize/UnifyDTypeScale.ipynb @@ -0,0 +1,146 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# UnifyDTypeScale\n", + "\n", + "参考：`tvm/src/relay/quantize/realize.cc`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```c++\n", + "float ChooseDomScale(const std::vector& nptrs) {\n", + " if (nptrs.size() == 2) {\n", + " // x = a * s1, y = b * s2\n", + " // x + y = (a * s1 / s2 + b) * s2, if s1 > s2\n", + " // = (a + b * s2 / s1) * s1, if s2 > s1\n", + " float s1 = GetScalarFromConstant(nptrs[0]->dom_scale);\n", + " float s2 = GetScalarFromConstant(nptrs[1]->dom_scale);\n", + " return s1 > s2 ? s2 : s1;\n", + " } else {\n", + " const QConfig& cfg = QConfig::Current();\n", + " float scale = cfg->global_scale;\n", + " return scale / std::pow(2.0, cfg->nbit_activation - 1);\n", + " }\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为 `ChooseDomScale` 的函数，用于选择两个节点中较小的一个作为它们的共同量化比例。\n", + "\n", + "函数接收 `QRealizeIntExprNode` 指针的向量 `nptrs` 作为参数。如果向量的大小为 2，则根据两个节点的量化比例计算它们的和，并返回较小的那个比例。具体来说，如果 `s1 > s2`，则返回 `s2`；否则返回 `s1`。\n", + "\n", + "如果向量的大小不为 2，则获取当前的量化配置（`QConfig::Current()`），并返回全局比例除以 2 的 `cfg->nbit_activation - 1` 次方的结果。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{dropdown}\n", + "```c++\n", + "\n", + "/* \\brief Unify the dom scale of arguments */\n", + "Array UnifyDTypeScale(const Array& ref_args, const Array& args,\n", + " DataType* dtype_ptr, Expr* scale_ptr,\n", + " DataType dtype = DataType::Void()) {\n", + " static const Op& simulated_quantize = Op::Get(\"relay.op.annotation.simulated_quantize\");\n", + " const QConfig& cfg = QConfig::Current();\n", + "\n", + " std::vector nptrs;\n", + " Array ret;\n", + " for (auto arg : args) {\n", + " const auto* nptr = arg.as();\n", + " ICHECK(nptr);\n", + " nptrs.push_back(nptr);\n", + " ret.push_back(nptr->data);\n", + " }\n", + "\n", + " // unify the data type\n", + " ICHECK_EQ(ref_args.size(), args.size());\n", + "\n", + " if (dtype.is_void()) {\n", + " if (ret.size() == 2 && nptrs[1]->dtype == cfg->dtype_input) {\n", + " dtype = cfg->dtype_input;\n", + " } else {\n", + " dtype = cfg->dtype_activation;\n", + " }\n", + " }\n", + "\n", + " for (size_t i = 0; i < ret.size(); ++i) {\n", + " auto ref_arg = ref_args[i].as();\n", + " if (nptrs[i]->dtype != dtype) {\n", + " ret.Set(i, Cast(ret[i], dtype));\n", + " } else if (ref_arg && ref_arg->op.same_as(simulated_quantize) &&\n", + " ref_arg->attrs.as()->kind == kQInput) {\n", + " auto new_arg = Cast(ret[i], cfg->dtype_input);\n", + " new_arg = StopFusion(new_arg);\n", + " ret.Set(i, Cast(new_arg, dtype));\n", + " }\n", + " }\n", + "\n", + " // unify the dom_scale\n", + " float s = ChooseDomScale(nptrs);\n", + " Expr dom_scale = MakeConstantScalar(DataType::Float(32), s);\n", + " for (size_t i = 0; i < ret.size(); ++i) {\n", + " float cur_s = GetScalarFromConstant(nptrs[i]->dom_scale);\n", + " ret.Set(i, MulAndDiv(ret[i], cur_s, s, dtype, ref_args[i]->type_as()->shape));\n", + " }\n", + "\n", + " *dtype_ptr = dtype;\n", + " *scale_ptr = dom_scale;\n", + " return ret;\n", + "}\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这段代码定义了一个名为`UnifyDTypeScale`的函数，用于统一参数的数据类型和量化比例。\n", + "\n", + "函数接收5个参数：\n", + "- `ref_args`：参考参数；\n", + "- `args`：需要处理的参数；\n", + "- `dtype_ptr`：指向数据类型的指针；\n", + "- `scale_ptr`：指向量化比例的指针；\n", + "- `dtype`：默认为空的数据类型。\n", + "\n", + "函数首先获取当前的量化配置（`QConfig::Current()`），然后遍历`args`中的每个参数，将其转换为`QRealizeIntExprNode`类型，并将其添加到`nptrs`向量中。同时，将每个参数的数据部分添加到`ret`数组中。\n", + "\n", + "接下来，函数检查是否需要统一数据类型。如果`dtype`为空，则根据`ret`的大小和`cfg->dtype_input`的值来设置`dtype`。否则，使用给定的`dtype`值。\n", + "\n", + "然后，函数遍历`ret`中的每个元素，并根据以下条件进行转换：\n", + "- 如果当前参数的数据类型与`dtype`不同，则将其转换为`dtype`类型；\n", + "- 如果当前参数是模拟量化节点且其属性为输入类型，则将其转换为`cfg->dtype_input`类型，并停止融合操作。\n", + "\n", + "最后，函数调用`ChooseDomScale`函数来选择两个节点中较小的一个作为它们的共同量化比例，并将结果存储在`dom_scale`变量中。接着，遍历`ret`中的每个元素，将其乘以当前比例和共同比例之间的比值，并将结果存储回`ret`数组中。\n", + "\n", + "最后，函数将`dtype`和`dom_scale`分别存储到`dtype_ptr`和`scale_ptr`指向的位置，并返回`ret`数组。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/realize/common.ipynb b/doc/read/relay/quant/realize/common.ipynb new file mode 100644 index 00000000..852cb0e4 --- /dev/null +++ b/doc/read/relay/quant/realize/common.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 自动量化实现常用函数和类\n", + "\n", + "参考：`tvm/src/relay/quantize/realize.cc`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/media/pc/data/lxw/ai/tvm-book/doc/read/relay\n" + ] + } + ], + "source": [ + "%cd ..\n", + "import testing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## QRealizeExprNode & QRealizeExpr\n", + "\n", + "```c++\n", + "class QRealizeExprNode : public TempExprNode {\n", + " public:\n", + " Expr data;\n", + " static constexpr const char* _type_key = \"relay.quantize.QRealizeExpr\";\n", + " TVM_DECLARE_BASE_OBJECT_INFO(QRealizeExprNode, TempExprNode);\n", + "};\n", + "\n", + "class QRealizeExpr : public TempExpr {\n", + " public:\n", + " TVM_DEFINE_OBJECT_REF_METHODS(QRealizeExpr, TempExpr, QRealizeExprNode);\n", + "};\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这是两个C++类的定义，它们分别表示量化表达式节点和量化表达式。\n", + "\n", + "1. `QRealizeExprNode` 类继承自 `TempExprNode` 类，表示一个量化表达式节点。它包含一个 `Expr` 类型的成员变量 `data`，用于存储量化表达式的数据。同时，它还定义了一个静态常量字符串 `_type_key`，用于表示该类的类型信息。此外，它还使用了 `TVM_DECLARE_BASE_OBJECT_INFO` 宏来声明基类对象的信息。\n", + "\n", + "2. `QRealizeExpr` 类继承自 `TempExpr` 类，表示一个量化表达式。它使用了 `TVM_DEFINE_OBJECT_REF_METHODS` 宏来定义对象引用方法，该方法将 `QRealizeExpr` 类与 `QRealizeExprNode` 类关联起来。这样，可以通过 `QRealizeExpr` 对象间接访问和操作 `QRealizeExprNode` 对象。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## QRealizeIntExprNode & QRealizeIntExpr\n", + "\n", + "````{dropdown}\n", + "```c++\n", + "class QRealizeIntExprNode : public QRealizeExprNode {\n", + " public:\n", + " Expr dom_scale;\n", + " DataType dtype;\n", + "\n", + " void VisitAttrs(tvm::AttrVisitor* v) {\n", + " v->Visit(\"data\", &data);\n", + " v->Visit(\"dom_scale\", &dom_scale);\n", + " v->Visit(\"dtype\", &dtype);\n", + " }\n", + "\n", + " Expr Realize() const final;\n", + "\n", + " static constexpr const char* _type_key = \"relay.quantize.QRealizeIntExpr\";\n", + " TVM_DECLARE_FINAL_OBJECT_INFO(QRealizeIntExprNode, QRealizeExprNode);\n", + "};\n", + "\n", + "class QRealizeIntExpr : public QRealizeExpr {\n", + " public:\n", + " TVM_DLL QRealizeIntExpr(Expr data, Expr dom_scale, DataType dtype);\n", + "\n", + " TVM_DEFINE_OBJECT_REF_METHODS(QRealizeIntExpr, QRealizeExpr, QRealizeIntExprNode);\n", + "};\n", + "```\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这是两个C++类的定义，它们分别表示量化整数表达式节点和量化整数表达式。\n", + "\n", + "1. `QRealizeIntExprNode` 类继承自 `QRealizeExprNode` 类，表示一个量化整数表达式节点。它包含三个成员变量：`dom_scale`、`dtype` 和 `data`，分别用于存储量化整数表达式的域缩放因子、数据类型和数据。同时，它还定义了一个静态常量字符串 `_type_key`，用于表示该类的类型信息。此外，它还使用了 `TVM_DECLARE_FINAL_OBJECT_INFO` 宏来声明基类对象的信息。\n", + "\n", + "2. `QRealizeIntExpr` 类继承自 `QRealizeExpr` 类，表示一个量化整数表达式。它使用了 `TVM_DLL` 宏来声明类的导出方式，并使用 `TVM_DEFINE_OBJECT_REF_METHODS` 宏来定义对象引用方法，该方法将 `QRealizeIntExpr` 类与 `QRealizeIntExprNode` 类关联起来。这样，可以通过 `QRealizeIntExpr` 对象间接访问和操作 `QRealizeIntExprNode` 对象。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `QRealizeIntExprNode::Realize` & `QRealizeIntExpr::QRealizeIntExpr`\n", + "\n", + "```c++\n", + "Expr QRealizeIntExprNode::Realize() const {\n", + " Expr data = this->data;\n", + " // dequantize\n", + " data = Cast(data, DataType::Float(32));\n", + " data = Multiply(data, this->dom_scale);\n", + " return data;\n", + "}\n", + "\n", + "QRealizeIntExpr::QRealizeIntExpr(Expr data, Expr dom_scale, DataType dtype) {\n", + " ObjectPtr n = make_object();\n", + " n->data = std::move(data);\n", + " n->dom_scale = std::move(dom_scale);\n", + " n->dtype = std::move(dtype);\n", + " data_ = std::move(n);\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这是两个C++函数的定义，它们分别表示量化整数表达式节点的实现和量化整数表达式的构造函数。\n", + "\n", + "1. `QRealizeIntExprNode::Realize()` 函数是量化整数表达式节点的实现函数，它首先将数据类型转换为浮点数，然后将数据乘以域缩放因子，最后返回结果。\n", + "\n", + "2. `QRealizeIntExpr::QRealizeIntExpr()` 函数是量化整数表达式的构造函数，它创建一个 `QRealizeIntExprNode` 对象，并将传入的数据、域缩放因子和数据类型赋值给该对象的成员变量，最后将该对象赋值给 `data_` 成员变量。" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 自动量化 ForwardOp\n", + "\n", + "```c++\n", + "inline Expr ForwardOp(const Call& ref_call, const Array& args) {\n", + " return Call(ref_call->op, args, ref_call->attrs, ref_call->type_args);\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "这是一个C++函数，名为`ForwardOp`，它接受两个参数：类型为 `Call` 的常量引用 `ref_call` 和类型为 `Array` 的常量引用 `args`。函数的返回类型是 `Expr`。\n", + "\n", + "函数的主要作用是将`ref_call`中的算子、属性和类型参数传递给新的 `Call` 对象，并将 `args` 作为新对象的参数。最后，返回这个新的 `Call` 对象。" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/read/relay/quant/realize/index.md b/doc/read/relay/quant/realize/index.md new file mode 100644 index 00000000..dc4078aa --- /dev/null +++ b/doc/read/relay/quant/realize/index.md @@ -0,0 +1,9 @@ +# 自动量化实现 + +```{toctree} +common +GetFixedPointMultiplierShift +MulAndDiv +UnifyDTypeScale +CastDtypeInputRealize +``` diff --git a/doc/read/tir/q_multiply_shift/intro.ipynb b/doc/read/tir/q_multiply_shift/intro.ipynb index aa78f89e..3087ab3b 100644 --- a/doc/read/tir/q_multiply_shift/intro.ipynb +++ b/doc/read/tir/q_multiply_shift/intro.ipynb @@ -147,11 +147,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "int16_t q_add(int16_t a, int16_t b)\n", @@ -170,11 +166,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "int16_t q_add_sat(int16_t a, int16_t b)\n", @@ -210,11 +202,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "int16_t q_sub(int16_t a, int16_t b)\n", @@ -233,11 +221,7 @@ { "cell_type": "code", "execution_count": 4, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "// precomputed value:\n", @@ -247,11 +231,7 @@ { "cell_type": "code", "execution_count": 5, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "int K = 1 << (Q - 1);" @@ -260,11 +240,7 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "// saturate to range of int16_t\n", @@ -279,11 +255,7 @@ { "cell_type": "code", "execution_count": 7, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "int16_t q_mul(int16_t a, int16_t b)\n", @@ -311,11 +283,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [ "int16_t q_div(int16_t a, int16_t b)\n", @@ -336,11 +304,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -381,24 +345,9 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "-32768" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 10, + "metadata": {}, + "outputs": [], "source": [ "// 《Fast Inverse Square Root》\n", "float Q_rsqrt( float number )\n", @@ -427,11 +376,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "vscode": { - "languageId": "c++" - } - }, + "metadata": {}, "outputs": [], "source": [] } @@ -446,7 +391,7 @@ "codemirror_mode": "text/x-c++src", "file_extension": ".cpp", "mimetype": "text/x-c++src", - "name": "C++14", + "name": "c++", "version": "14" } }, diff --git a/doc/chaos/tutorials/transform/SimplifyInference.ipynb b/doc/read/transforms/SimplifyInference.ipynb similarity index 100% rename from doc/chaos/tutorials/transform/SimplifyInference.ipynb rename to doc/read/transforms/SimplifyInference.ipynb diff --git a/doc/read/transforms/InferTypeLocal.ipynb b/doc/read/transforms/chaos/InferTypeLocal.ipynb similarity index 100% rename from doc/read/transforms/InferTypeLocal.ipynb rename to doc/read/transforms/chaos/InferTypeLocal.ipynb diff --git a/doc/read/transforms/defuse-ops.ipynb b/doc/read/transforms/chaos/defuse-ops.ipynb similarity index 100% rename from doc/read/transforms/defuse-ops.ipynb rename to doc/read/transforms/chaos/defuse-ops.ipynb diff --git a/doc/read/transforms/div-to-mul.ipynb b/doc/read/transforms/chaos/div-to-mul.ipynb similarity index 100% rename from doc/read/transforms/div-to-mul.ipynb rename to doc/read/transforms/chaos/div-to-mul.ipynb diff --git a/doc/chaos/tutorials/relay/transform/function.ipynb b/doc/read/transforms/chaos/function.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/function.ipynb rename to doc/read/transforms/chaos/function.ipynb diff --git a/doc/chaos/tutorials/relay/transform/fuse-ops.ipynb b/doc/read/transforms/chaos/fuse-ops.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/fuse-ops.ipynb rename to doc/read/transforms/chaos/fuse-ops.ipynb diff --git a/doc/chaos/tutorials/relay/transform/index.md b/doc/read/transforms/chaos/index.md old mode 100755 new mode 100644 similarity index 55% rename from doc/chaos/tutorials/relay/transform/index.md rename to doc/read/transforms/chaos/index.md index 489b6235..bff6e1fd --- a/doc/chaos/tutorials/relay/transform/index.md +++ b/doc/read/transforms/chaos/index.md @@ -1,8 +1,9 @@ -# Relay 变换 +# 解读 TVM 变换 ```{toctree} -:maxdepth: 2 - +InferTypeLocal +defuse-ops +div-to-mul module function sequential @@ -10,4 +11,4 @@ print-ir instrument pass fuse-ops -``` \ No newline at end of file +``` diff --git a/doc/chaos/tutorials/relay/transform/instrument.ipynb b/doc/read/transforms/chaos/instrument.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/instrument.ipynb rename to doc/read/transforms/chaos/instrument.ipynb diff --git a/doc/chaos/tutorials/relay/transform/module.ipynb b/doc/read/transforms/chaos/module.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/module.ipynb rename to doc/read/transforms/chaos/module.ipynb diff --git a/doc/chaos/tutorials/relay/transform/pass.ipynb b/doc/read/transforms/chaos/pass.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/pass.ipynb rename to doc/read/transforms/chaos/pass.ipynb diff --git a/doc/chaos/tutorials/relay/transform/print-ir.ipynb b/doc/read/transforms/chaos/print-ir.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/print-ir.ipynb rename to doc/read/transforms/chaos/print-ir.ipynb diff --git a/doc/chaos/tutorials/relay/transform/sequential.ipynb b/doc/read/transforms/chaos/sequential.ipynb similarity index 100% rename from doc/chaos/tutorials/relay/transform/sequential.ipynb rename to doc/read/transforms/chaos/sequential.ipynb diff --git a/doc/chaos/tutorials/relay/transform/utils/helper.py b/doc/read/transforms/chaos/utils/helper.py similarity index 100% rename from doc/chaos/tutorials/relay/transform/utils/helper.py rename to doc/read/transforms/chaos/utils/helper.py diff --git a/doc/chaos/tutorials/transform/custom-pass.ipynb b/doc/read/transforms/custom-pass.ipynb similarity index 100% rename from doc/chaos/tutorials/transform/custom-pass.ipynb rename to doc/read/transforms/custom-pass.ipynb diff --git a/doc/read/transforms/index.md b/doc/read/transforms/index.md old mode 100644 new mode 100755 index 9e0b1f01..11d3d625 --- a/doc/read/transforms/index.md +++ b/doc/read/transforms/index.md @@ -1,7 +1,12 @@ -# 解读 TVM 变换 +# TVM 变换 ```{toctree} -InferTypeLocal -defuse-ops -div-to-mul +:maxdepth: 2 + +intro +pass +infra +SimplifyInference +custom-pass +chaos/index ``` diff --git a/doc/chaos/tutorials/transform/infra.ipynb b/doc/read/transforms/infra.ipynb similarity index 95% rename from doc/chaos/tutorials/transform/infra.ipynb rename to doc/read/transforms/infra.ipynb index 3df4afd3..1eb08806 100755 --- a/doc/chaos/tutorials/transform/infra.ipynb +++ b/doc/read/transforms/infra.ipynb @@ -6,7 +6,7 @@ "source": [ "# TVM Pass Instrument\n", "\n", - "参考：[如何使用 TVM Pass Instrument](https://daobook.github.io/tvm/docs/how_to/extend_tvm/use_pass_instrument.html)" + "参考：[如何使用 TVM Pass Instrument](https://xinetzone.github.io/tvm/docs/how_to/extend_tvm/use_pass_instrument.html)" ] }, { diff --git a/doc/chaos/tutorials/transform/intro.md b/doc/read/transforms/intro.md similarity index 100% rename from doc/chaos/tutorials/transform/intro.md rename to doc/read/transforms/intro.md diff --git a/doc/chaos/tutorials/transform/pass.ipynb b/doc/read/transforms/pass.ipynb similarity index 100% rename from doc/chaos/tutorials/transform/pass.ipynb rename to doc/read/transforms/pass.ipynb diff --git a/doc/read/transforms/set_env.py b/doc/read/transforms/set_env.py deleted file mode 100644 index faddc21c..00000000 --- a/doc/read/transforms/set_env.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys -from pathlib import Path -ROOT = Path(".").resolve().parents[2] -sys.path.extend([f"{ROOT}/tests", f"{ROOT}/src"]) -# # from tools.tag_span import _create_span, _set_span, _verify_structural_equal_with_span -from tools.torch_utils import verify_model \ No newline at end of file diff --git a/doc/tutorials/auto-quantize/configs/create_config.py b/doc/tutorials/aq/configs/create_config.py similarity index 100% rename from doc/tutorials/auto-quantize/configs/create_config.py rename to doc/tutorials/aq/configs/create_config.py diff --git a/doc/tutorials/auto-quantize/configs/model.toml b/doc/tutorials/aq/configs/model.toml similarity index 100% rename from doc/tutorials/auto-quantize/configs/model.toml rename to doc/tutorials/aq/configs/model.toml diff --git a/doc/tutorials/auto-quantize/configs/set_env.py b/doc/tutorials/aq/configs/set_env.py similarity index 100% rename from doc/tutorials/auto-quantize/configs/set_env.py rename to doc/tutorials/aq/configs/set_env.py diff --git a/doc/tutorials/auto-quantize/configs/set_tool.py b/doc/tutorials/aq/configs/set_tool.py similarity index 100% rename from doc/tutorials/auto-quantize/configs/set_tool.py rename to doc/tutorials/aq/configs/set_tool.py diff --git a/doc/tutorials/auto-quantize/custom.ipynb b/doc/tutorials/aq/custom.ipynb similarity index 100% rename from doc/tutorials/auto-quantize/custom.ipynb rename to doc/tutorials/aq/custom.ipynb diff --git a/doc/tutorials/auto-quantize/fixed-point-multiply.ipynb b/doc/tutorials/aq/fixed-point-multiply.ipynb similarity index 100% rename from doc/tutorials/auto-quantize/fixed-point-multiply.ipynb rename to doc/tutorials/aq/fixed-point-multiply.ipynb diff --git a/doc/tutorials/aq/images/ap-intro.png b/doc/tutorials/aq/images/ap-intro.png new file mode 100644 index 0000000000000000000000000000000000000000..e9de1b061e5b63b2df04d73b70088fdd2333c605 GIT binary patch literal 119442 zcmeFZ_g~Y?w?2xxH+I>Af>KpLq@%RZZPZXidJ70b=n#5GHY!F0q=gnhO6XMxB{Wfr zw9tDs^xg@bJHfrr=X~$?b?!fKPy8hy@661aHM7=w*7F4ULRt3eCE815WMo(6<(|JJ zBRlU&Mt1J;g|px{xh@bd>B^9O=$ z1~=~NRdYj8xd((3DC0EpbI5}o6KMM+(W?4KU5Jkd7gQ=C(T7Q5U&6JL0!)SUdPBJ~ z!T$R#8QFus>u&$^{>v3g_&;yoJd#77`R9G}9R&G5Zy)&o?;roq)_C(>oH4QcS78v& zhB;3)+v@F0oJ@X{o5!+HsasYROsYzGX7fCnF|lo$0in;C{lf&SrV#amIbm2ey?z-NDs~MMg4s4qh6>-OVW{y8!ueHJ{~o6OBJ?_8y{zc1%nTogQC8fjx_P++@`aSqHF%-e=KxK!%x zj^SI7NG-)) z9&Ml9b}fz#bR1Hl&kh`im6#~`-MVn=Y6{;mPc_TfA^ivp88VVt z6+`QnAf7KvE!=sE+6HnMtS(*RDHdg5RC1BZ=Y7#7Q2a3ET@JU9m8$V%=ihmGcLOvj z-Yg&OJS&f{?0BfBr&5Y4)jp(^I2sgMz&G{T8N{^9(Zpe=<0}Jo+EO%WR&Smm16Cq? z-n_a=1G643i*U?Te=)2%-LJ82o^!rrdr$~{&q`AkgW7>$w{soiT0C>U1Zrt=mlt)W zVnZZ2pr7KjrXJ$zeQaO}p5+Dp(g9?rA2<^GV?_j46Bd(-TTs;XH=`c>nJvtHPeDz? z=BN4XVjQ;f2-k*p?cT0T--x1 zh-SkdFM}doQZ>&R@=3X368}&N!g65UA0HX=3CGg6=w#}|YYaeAcUES%cZ2IS9-=Y} zDR(s8F2w)-kWVeE6!eozW`h0iTJLKRzim)ou8r}q8kMig;$?$a6LS=-W}MaS43>+t z_Q|3{og!5gt0H|;QKg||WTZ==K^5JNPgPy+O{P<~Akr1!Q1=(*+SCmiFX4JT$v#yK z)N*Dlc+9$m_vZC+X7*`p^Z8^}Wwqek*e;&#e&X<^443uFi&CXgUT%Zj=ic8 z<$L^B)uH)HL8JM7M$_2t$CwDG?81X`^cdLdTQxl}J*^or^@Eu6r*?gF8*#S?IiD<) zRq|EVc{JbZr@VGr(2#HKmxgl_eL0#iv8z4FCPB|q7AiiN>O48m93_-B_mXav?bHTu z27V6VaFmpkeK|T*G&D39-Wu+;*_INwx24NC^ZSRpyZK$%N~bpWODn@0^;Map_=ePy zaLR532i{jPW|kcbn)L{kzK2G$8r_QV*|%Tx?{Y2;tAGEA2^zxY$vKaL+Z0%;u zuIEbmY%pCN`pJ;wqemMvadwhB116g@ZB)Zj`<)?@2TMgUF)@=Xb(8B{dL>JAywRk%=Q+7{Hi_5+|^EchD$;3l*@Kf6H*oVHB#wbf_b=9o4wIe z;>S)Z#^D+ai<@>^p$_F{ zU(&L$O{vMsz)}7Rx5%WtT%nE_2}iU&R^rYH9heI?;yxId575f?yj@hRuEpb;nQ?H2 z>@ndA8Er&(ICI<8tnBP(rM?HlucR1@t=eD6emRMOM#*f*DYF`wqB1ZXZNeEDR6@=nlwc+$Xj=_n zjS`OB7%e5Jsep*O!12uMMbb28`D4|4j>t~E@2aTJVVQv5h0*B9NY*xL4i3oz+Wf{z zoih8Rf#Z!9-WIy{1PM+-WhJGn!+V!`EAMzS@rP~dUt8sYgAqkNU#2H3YY$o9XYPK< zt<;rCC}TsoA6CAm&DQnTke*ycreHtm=w?j@#;I!Jiz*06SdKU7T0dw+|YseR_A zdn+YYU6!bol|hxjvC7KatsNsfTIf3xeoN^Zj5nyae(222pc;gp0?ELhLO1@~@Pt6# zv4;&{ZI=<@#SV#$lNT!j2d$4=;gZa;xq79_V`bKZU;7Pxy+;p8lcFNVp);{+B|ohKzK+ZiC^g#@MqY>jGmR|(4a$2{g+odK185^#Zx-V+}4dPR@UpNR6ZHZ zs5opd^uiO|Wn%3rcZRH}1TRlIxkcZ@XL8ci>#`~SWCov!B@TmCxNfe*v*2~Ji@O-X zrmQ+n!`gcH>!bcXXK%{mAT8JEA$OG6z`p#j-SDX5)XdraDJyx;w(8{gpi$@!;yxmH zP+5+kNDx10l%RxD&fc02V8OHCRi&<+J-spc80r>nPS5fk&g^zVm#uG8`CbrC~4st0?r;A!)E*je$N z3t(Eh*`#+8|ICb51%2dGQ><(F^y&99SNE+>+4@}H!%3#;9Tkb~h=Gl%a1~`A`P-yL zMK8ZC{0!k$u;%Kzs-#Y{NFEa9WY=ts+@BN=vx{KtN^o3`l}D#Ij4+c766!-T2>tet zJExxTWxY;kcfY@eN62T{%8<@8ew`nVmm7~)R8)*8SsE;oJ={gAco~asUL_4R=ki@! z4jsiJIJSY7iQ}{z;dks{MYw)#w~hwRnrO4WxT-nj$E!!u`KwonS@Sh zwIgnWXUZ0Addj6?5LsfGENMB+N1sW1wGi-xj0_YHj}MSVYs2Hmnz7EUiBcz-=?NYn zN-}!P-L+`(r;yC(U&NTDq@;i-Suv>@8ykDNvE@_^x&*HWrH;vwG^y76j6)cyG-p2VJsDfMm`1k+az+f;ft^QVTv9LOK9yWYo$3=}$h+%sm!%P33u7rX(m)E5?1!8GGF*EpI z^A5?atfx%uE`Kof^{q~%ck=sXMhgn}?fdJK>tQPtlH{X>f9H*Os5!QqL-Uuvj)kfB zu&#)y61aAF6L3#QUwGD3^%+U1_119d&>VmX(*t*(btBM=0n@r)yN@q)x%J0sxPMGg zY_lYYUseZ8_k6laG>qE`Y&wbEn&}Yhqmykost{GWHvh;U8tybHbgKY2b!6RaJ})HJ zSaO>dvHH`NE)J5&v0Bw7N#Ex-u%hGoGJmJe9YwN&EDf*}|L?4p68^Ey9nXpf`;cu{ zS0PAlHK#<7wWyH#YmU(YB@~xE^jJQgN*{`&9^ zD`+Nw*xB>CAC9bx`KU?Anb?JrTv@H^`fndt3=0dpboug4ozW05$32(co*sHJ7wgxr zpNAtipC9_Y$@6qtb@%tmx6!e2X3$hE*E;`$xnDBQDZj&%r$Gr_ubdi09mmGSHRfA; zXH>x4Ie--p3YLQL2P-_I#Yc*9B~nlOlcMU0bPTH1?J&QTJCmTg zr*D?u&Xm|K(v+u(5lWBR{<&-y0|{*CY9OQ~6?*C=bQKdk-m&6oEYII>E+ZQ7nM)$Z z91K)dJGZ2qNtaZ@Qvw$*@ns^8GoRU>noQl>CQcsMWK}c&CeS$(d`aHAu`|#=G zn4YAv2tpDeQdw9H2X6KqNyV17mq`a4aI=RJKCf1qm&Zd1=U6}%{mtY@b{Z^L@K3~C zmdq_J@13q4g_iJi5ru1I-a}AhA8pT^nCcb%9lDOeJ*hdFu_(D#ns3s9$TKqu|74Bq_uI$oUx zPsNJwO1EumYh%IpEV1FcPfqt$C$mmxpu}caK{eLCF8 z&7p8%UX9YlC9DuQd*);kE6#_uF%UMI^CpYB{({zQVnxH4IPx4G%2|0zcvBBgngS`# z9s#fUy=|P)5YCNy=6|+KWTe*ryq)XmAaH5QS$5vV9B{x}=KFxZWcsx;X?q-{PTD&1 zJsCr!aTl+b>HRT7Oi7w!8GPMR&Zd==UV+j?2ZkSvquQ^t#OgDV$s~a zrAVKXZGIOCSPl9)I6Datm0xP51M+C;lk54b&z-By?qJHqmWqOgbCqJayT&d#KP$CH zf3}OD1;!Reej68c{B;@7{M$IFhW{-E^(g%h;>afp~0X z^jd0I5X1)q=O`lP+cv6p(86xs!cm9|23@au<|Bov1<2c9;PX}}M&l|q8@gV{d!60Z z&b+I`h}`6+njJps;dY^%y&#rU-Cx3O^|5@@56aM`>~;K&n~p|^TujfT|0g%Esg51} zT|~lAEO}8lg#Q>nv`Z(3+>EGk`vW9-o997wc+^sJ*uqe{0kV|Tv@}u*crI22{jRuJ z03_zj$j#29$dtY~wvRXi*t3?;Nxk%<;q*dcbvcl}vSbt z;JGCPFKK?|^lI|6*X?gD9)YtmG+|WmOl7T|K{S(G<~~=Rf0Ss$)m=HX9s75cQS~Uu zW#((ESv_VTNmN2(R7UUVcNf29lK(y7rBaKJo~Xc z&kYSQDjV(CaHj&@get?I*0f@iZF1EdX)DPtx!j7MvPBa+)wOz=6Ui1g_a;jsEF}>r zd|p;gj=D5KysiGPl0*x!FuM`g+5*vKvukA>xgvMl-T2Z?q!rW#3_vW&m$!qVA^5Q| zi6p6rVfTrbvU6uUva&%=3KnNuZm(l~AD>L|p_(OXjypMIx;Ji$+_Vj>URxQ>e9~i+ zMg^9x!{}?mhp5gCc}%L;kKe5N5G#_-tURNT*y^u27G!2lDJ?BcO@002{nd8yO+|Bm z0I;N|q)e9hylb4Czx?p1>E+y5n!SZHCH%28k72df6AMq`N)VspV6}IJa(bXGL0^>~ z?HosLZA=L%@r%AU_DRDS$=I)ZWfAw`Uw4 z9wMvJ54(|@$8D^VhHF}ZI^)mV_C5U<2dE7!FIUV@lza_Lk6X<0__T}~YGzztJ*W4h zT@`>;b1yZEGE5Hgz2TYyKgEW-`m8NGuflki1}X}*ZDZWNC|Jg9j6pWL)a11=dPM6o z7n+p2(W`ZKyqB%m+w8nyC;Ro++uWp)%P;-kNZm<54a%dL#hKh?7(^UihndF9qXp3r zSs%%b*16DS5bweq}<_hzL|_S#P-&P(lr za^${}9Vw+6hXNoZ=wN^UDbcL-Je!%0j?P8{8jZGEomgC?;e{Q7*hF+pJQyKX6RS>+ z#!t9;c@yUbGcq!EN&f@zo!J&puck|Gv#(FoZ0Fh??X~-&PF_;NjqvyJPc1DM%34I* zoe28&NgDgoN$Jc%xw{NeDy)4-QQXPQqrkaWqg+hWZD<^|6 z)g98eFX={;-=7%jrC}N4ld3rcu|~X?c(2srh*_lx+|e5=N1c^}OCwE=-(y6W%*(m@ z7cZ7X)FgSe+|GZ{{n);66a+v=HUz5LNV2mukQAF!b~EE}l(P{8+f2XCsSBgk4^uFW z^wmBe`e#*mRzTVFZbi7MUL7ugFguJtS{Fu9&B{rh<|cdIA~e^ohMr7H9h-zT%+Jp! z1(A?Fg<%^Zy#d-bUw|`BGp&M(i)e=pb~H97{Qc+U@0x)=b4dYp;wO6J`>0CkD{C8v zz9Y_P-;;y!7P=^2!-TjvF&FQJ5U(j_y;9q&2i@=AzyH3m#YeLAvO~EX+OHf9fwqVe ziNtmX!pI9MePjkp>5vFm)@CtEA) zE1e-#weF}`I>b>eYc^@z4x^m@c9fd4%Bl4-@4<`I6e0GWw<8=v#%XYp837#vrT}xfZ+~IS?X94fu zC%6%i1ww>Z0XFWNZNlX@%lH8W$w#S7XvH}*aM z)|zM%NvX-by;`3$W5oBmpcPxE6BZdcz3z5eqECTbMiCt{Mc1Fu(}HDBnVy=4d8@2l zQ3@C?y?-<#$PtIjXD~VE_t?@Rq!OX0Is9RA2+_58xzh8g3IGWbaY0r8IrLfg5W$Dd zC^E9v(Yief-`!{C8J0qEY<4I9Ooxwc59d{UJx1#G0A}nqh1~r4^CyT9H->geST%4o zvDL~=fWf>stuBp(K!L%^YmS!f006u5{oUnfr6=ttharcuHsjR>W0H1bm1SjRV`El! zpeKOy+5N%PEbM)}naG9!l_o2I5jPOlAT+r%LYnS{g{wsJt1bMnr3{D9r!l4oS~o0i z@4Nb}5#%4X?Q0Ck&cIGzjZ21OES9BE5m2{v&TivFu z1at!89r+O`lrtkEBdOL1es>x)66z*Cc=iv*!5^>I+`1CVsmD#lU8|Z(2W-Ir$D=PE z_1MUeqU0cxZ*LexW$QXXxj^AVY9&lfZJ96~3~&#hENInSYO}%Iw$mnX0er=`Emjbm zs}kK@0|SF_7CgY=J_`eU_|BH)730!&(1LKHoRvEKMX@oq_>BE>VlwVm&4P|Yr& z>9aQ-_3UJrssqtO+w3{`xfOgedTC_xNtRNYC zlMJSf!?*<&Sjq_O4Q( zg8SaF$9TIf%U5~jZwl68C@32aeT&7V*%+h)Y#0H26B-jU?@|G0`+ak1Nh6Eym{n9l z&y&(+-Ys!LyZ4G?sr8BP3DGy{VvbW7RGT0U+#xwRxj;-g0usQ}eF&N`_m^OMdwXii zr!am}Qc_1%F09I9>Eh7SCr@hY=~9d0OmE-5om7*NnTf2s9X*NM1Ysl}Rzr997MQ~xRTbV1azpAMgr&i+_hV}4LcK4 zvtBacyQ;+zKIpmO{Ad2PHKxN^hrlS(&bR-rdFazP{Z^tjl66`wuOWWr1WqbJeoH!8o!^{&KZ1iiYm`!q58Hh^5c*pqaizbz7URHt*Oa^2+! zMgkNX5nxez@9V$T*4BcXSJw6rY_=D;tVhZnfkkXfJ;y5DHyR495Eo7k+fT%?b$jVj zM;32M8TKK8rUoKpcs$T32_oLnr#dhas*BqF*Nl!bx_-%OqCxNj=(SIlh=5!PYjw?* zGh9f4Q+of%uZr&DkD=9Jb;`2B=GOy}f%g*Eo1vnpsgkH^x>R4L0bSZ1gUJ%Ja3{5+ z$gv9Ai7gSGy51bg`HYSiR_QiUR}N~`?{zwEyT1aa zgARzR6cte|)3}3`3XphR`{S&N`1;$*qvL~B-^JY0NBsPHy1HJHiJj&w2=ZiQdJQ}W z_l7%k!1QY}=r!OXoF-;8G#Y*4G!zY|snVX-{Ls@IWL6RUvC`!>Sy9+)YM_>|Zfg*; zy19?pOLw}Pa{N$Bf}`xOR{gCY|I$N#w%GNLP0OIENOTSvx$-2qzkQC1F&5(L;=(a@ zWV`q*qB~tBfO`ZWO0Jafb&m=gU!LqnA#OkM z=I=A(t*0e`B&h^2#5u{Yk3AThl&;PTtTmdD5CQPmo{;#-;bu}&)-6}I=CqSvOvg`b zUBuFMupypJjbkvrlX^q4k9H_1I>&|N=l%bzAt@iZ}gE( z3w)T--6sE^(fq8Mwzo81J&seEQ7B@O#2Xr51;J zUuP}y_R{Dj#i|*kDckuyRCq@3G&Q zYX0mwcvRprzvth*=c#66{@Qcz4n-^91fT}6!M)MX>IfRK2AiUtw=hflOxw?|_fW`9 z#Se#-^OKDsAr%pyJ*Z}h4*o40*Me}}* zZ=oQUN6luRM{4)5Vx`*xDY&#Q34VupGw96lZN@166TBG~S&Bq29hlq1I#KRf1K3$7Z1XDn#r z%#4;;@S*3uiawLzr-nPi%o^7;Hm+xQFOX(8}b^phd(E?AFQCw`>H&>RrQMG_xG2B zDx1cW=|-@54~hqS%qoHRL2S{PY>Wc~dple3O25ZO<{e`%x}(B#Y1HNh-4jw0AZ#ow zieK%EI4I#sFNB{HavJK~H{Qi*#6tpwlwH>#R33%ZfRF?_mJBNt^nYJDw^N|Do&FgT zI21D~xpH~P#N{JHgAp;%gW#oiW*pcDU>QQk4eEqSfNp>~X^j8WIN-jo+3JG2KA-87 zWy;OUx>R)lEOA#%>;RA==!mY~#ENn6jV3m5&8;jhR*<@M%M?Y~G zCvSQ+I+f6NM5io(uN@tK3+A_B!7nJFCkcf8g9GMqJuvByJ-Bd>b*UZzI}(_hSKL9H zbj5L*y?!(kBLN`2DgK<_d66!ug6pS*Bh+MN&)Y?lXhfI(a{LtHy<`&J*p^>&y#Mp3 z+yFy*4CFFE7fAi;A3ptoyw!6K<<$TK0C^P`mu|6nat)Ewa8F^*5tL2RU^QZh>d|6W zT_iuIsm|RpU9H}lzy(@h(qNwQXf-u8hHH&B%St9l{+so6?!=W*wF|qTv6D9Ql;Ab> z^E$OHIY?zW<@5Vbz`?2j!J^SDA9SF50NBz{)xSjrm$$OA0`*WA1{xj~Hsy*oJX+A2 z_Kb{(XnjuLVrLH%cVMSNH-KPg`#rIUR7ierx`yj|V5#Sgm zQV18%qqOU#TdM^($?pT;(afO2SkE*YQ#H7_0X7G%=a% z*^&9T4|9eIq+dmUg9IX1kCT(rJ)E?tzQ3zHiOqf+-o(*<09JWqNBgyuN*?b{h8r%? zk)@#^Dr`1)Nv$a#(5t#EoDG^=Qh;??Thnh;WlGSuqYq`C2|jaRT2=1YaaZ;S_O9n= zDtO^=#}Y&UK`ll+f0jBfy|Uf`<2~Rw*=*qbgQw7YU?$TV`(Z~@cFn@I%$xINET}5H zCcgO%8z_7r**vq(egF4dUm}ZkUFQU+NUe3vQoyRA@Sc0-kaO5(=d;<9spjwR&%nR{ z0|xA$ANy;y0GfegDB;p)B}wl)pq5kkQRqxW*T zZu#6+3NEPA$*{)06}qMb4#22>yt}kQ_({WH$KKVnjtc$dD+zk>^w7&zt;z&+Vxkll z^Y--f6Cq3@4$nhR-PnB}>FDAjbeZEUm=)IIU}u>)?t4k@L*hbTc+3HXdMYHQEp(b@h0BZ7uGy`;Bxbf;l^zp6IE5kIrE{b#D4UBV>& zh03@*CSin&hu5ov+m#5`A4CVSXQff|ZCyKCUlYp1A*%z0&(gcuzY3F}ERc^~BKJDo zp+eWyv8*hP-yLOat69O>Y&1ROpEmk(ws4kT}`VSKQNe?&d6q zX~(akoVo$*n=~94_kKo~1g3))f5Z@}*Xwwr(eM;)TL?NOUyv$M^V8<5GWG@EYNXe^qQ{!W#b%0n@dXQxCWLU(yDGC`N1^}@H$fN6LSyI z=SJXKkn^z*;_|)o7YY*S;E|WxvqU`qonWLI_eeLjVnBbafb+E01Ot)&{hqlTnw@gC zJQ@;cxkP)_dmstvXqqiOq&+H0?~*?hJh6SVW$V;?KZ5zjkcjjorY&h~MS@puUwjWJ z)e%&)y%uc*o~|8Qe`n(04yFk=IyW?TIoeoF!kO)<2q?P_3Yh}Urb5!7bN9uHR zU7qRcnQ#RR)9eeAYT^HDe!v+TK)|n}o-IQbIZ-+}wrBLyd_ED#Zyz5J#O=J5EWQmk z<#bLU5dSxu_z-gBxRtF(P`xfJ;}4EGk0peerAQQ!|7$IZp}&U(X8o5)qR@tPJc}%v z$%+R&3m69cL1*?h$!sJG{9X?GKcDq43j|PH{_{EiZqJ>DGBhmh18sH{U2g=0={FgU zTs<7pPe(pi3*C5pptz3RnV!0kB9KV9d1woI%Y4p~#Gk^Nif!2G(`XU_;o> zfvwghG4zrFoj7uj#OVxzk_=0lS>;J}|04WjM&ZZdms!b-(ds?YGHHd6V(BgCecB{a zDq*zQYSUmi*Bo}E)~=T}u+T<2=&8NPR?m}Ij}%2dH4-fP3h{t&DZ5>64_( zk80q^&Qg&+|7~>f<1Ij}1O)WW2Fg=@!#BaWsM$x*o|g=xTA9f!u342)pA=051r{`+ zME+QM(x8{I=Qu>k8f@52Tf2H9J^N|a^ZtFd0@A3U7m0e20-gkXVesU%{XtLH5xMLL z=}Ay>e7yGNq9^CAqTOIXa3QVos%UXGnnXZ7c%JeRXDA;Y^GOkkWsy%$UYXt4b!!sP z6eQ37=wSr1bwHBTX!g`{)B1e?Bcu z-X|2A--E=h-1~HYOoMb0o*^T3@8iLMS<=E8$t%Lvo1a51GJ|irWG<53axL^Hah&J0 zQ(7jqREgQy3$K7>HKLsVv&8qPyJkpTj*7P2JjUIXixToNW?3&Zvga<%YRl_C!vaPP z4maNMB_5D1d&k9^D-?e3NWWXvF+jJYA6TXUlCPElg(C0bUG{FptJ($&3qJ49h~>wc zEr7m#F9R9Tmdk9aOJ-SL4{+lCUF@AXFBvLFX&ZGV>Cy#UV&~Ky_HLPF3A-1QXY4@% z3=^}7u&B$$t!Tx&%8eq=~8q2 zsy~c{Av7jyQLb7QGs^qdg{2CkiJR)qAd1qKxT_DHv*N(r25kCBR+%yR-*&$+OWX@t z$~EfB7$fnU7Z^$WWz+IRtUc@z+C3b|samNj$$ z>kaD)G!Cw>4OvZMXcs&397UG<8f_47c5nhUWaD7RZt9JsN;) zcF@Zt4E=P!aiQ1@rgpP8xvn8B?Jm$ulnzIEl)VRfiAZ`wrPK6~oNS`cOVStg!>&Th zK7SGA(NI?#%PnNgSS`#I%VbkfYrziD+a{uL;~Cv?pW#!H8qzk+*7LUttTGS&JnPr; zjv)>;`-r6H&_(Hm!7uCr^#1f5KCWxM@2P3|qrtS*)1?~ zLfrE&fV7I7%!or@IlubV9ilZ9oG3saesz|uG^FC^fZfuU8Zx-u{v8;^rYUIu@)mii|b$Ra!Nb=B?BNeNE zzZ#VXLLN=`;GwSDfSu%OR?0@>O0)3ZqZMCnA1M^l*xzT(QPvUbQFh-JI*)R_<(snS zd~cod1I@h?FDt9?rj%S>m*14D4}O1j%({w-m#CIH4~SSJbX%`k-nSfBD$qqUYeu@_#IE?$7x=fhCTwY#>Rprmb%= zD}I^}`^qX;MAqhYQz!GK?z}(u@WCg(WBl4KX98@U1v;6ZxK2a%W}W1Bzh4ngNa5)8 z^@5xfSx#S%ZT+5eAE%9}KZgiDe4mvPIbl49p;^J0oh%Jy8h_w}12rWRD~<%2>KYXfKfzM+?W7)#q9P0cNTbt#S+j#c8uJ@NkW*wMQn z_Q5-j-3Xy8AJVnNo0j*J@Gelp_7A;;E%zhGl7qdYGuc17tq`>2OZ%|6JkR1JtJJYu zIoa_Mk65KJR~teb@uNT-1hMLK@B$~%E9Awu3Y+5!!LH+cfQeZT~k*4SJf!<53_(q{&b2rrwKu9 zoXM{Evj5e0`#Ds-2jiR`!g&)69Jrf?#rLZ$?VdQAEq>Do>BsCb4ww{E!>pmf`D5$0 zzi`q~84vmiP|rRZ3{xgI?-F0!QtYY^pr@Huv1VpifyW4S4K++D$@;7Gc0-&!GYbrJ-_ywP z$K%7`mR3HK^03Kl&rg~_^tqWKR$dktA%e0x%0zuND{9ZL9T}V@_R&IvQMP8$fukzE zm$hgF;6nz?vEpQ42SZl!Uw;amRv6Ih%wsYxIZKMRUmnRgUUoFVsy}Udzf&v2?JPB% zAvRsb*~-@Y5{Sl%w`|5A-}yXlq#KsX+yxSZ7RyDtzMfJEav9vqW>>!A%;YgnkJ7@@ zm(bDn=`8?8^5k7#P#IPYaBN7gk)aXfttHd{gEuHuDUl<)}Xe)HMTKB8O#0 zY3F6(rdITNMqAC!iZX$4^+g{u9z z`yJ(&nqWUgOk@gF>%jR>+1xydMV)+|NUWpHuY&rf8|0+)=8lVVbZoS67B2R{vS;{6 zUwJj*YP}*uub%w*Y>oJGblws(PGdU;e@^ozZ~jBRuKK~Yth@#M*_*y&Gn-698u$lk zW1?lbt=UP_xu=3Vf%^GEwg_vGTDHm_tY`T#c1S6!PPRCg}; z!KY@`eBO+l*GBzBE^5KU2!C4w2ul}6*ctd~N!7%gfvyg<+UKhL&X}SXrc7>-D$tMS z)9R5~9@kICotr59&hu5bO+CFSDQ~4;q^Gq26T{I3&&)pf^U#l!+?N-t>WwvR9vmuK z{@&uB@7~|I|8Z!~*ephlfJiS-57Rj``98j;I z{E_-SLK z^l8L4ul_Q>Z8+cCvwe3wMf=}kt}YB^-Mahsq}S*~th+rj->@X= z4Aca(njb%Pv%RS))C215>^!#vm|VlglWaf*`{~n_^lppN;qI<39{^cHgVbKWd`d@R z#L>{rwkI;t(7+4z^`w(>bC-Ly3?pfI4R8V*jM1Rt0p^MXWM1d)-{OnY!kN$#G z5?#^Puc^mss;bv>qES2HGj#`s_%&>qMN2>SK7TL&aRrhL>$22CUk($ z1Y1bl-rl~x^X(`ON&-nL`fq01(vnDZ{9o*WB5E@zt(uc0@N_o7Sk4EW@qNRd+`!@E z8T0Yldv?~*=f$+yj74fU{qs9)EKI(2X-AeM&=Ijd+r3&eT-qenWo9B**$TEFMkOo( zn#;z4a|R}>+lz)Lx+7by{Ov#Wd;mKvklU_uy=Ms zWMtkK52H}13!W1@y?>IRsL6J@Hu)S_HBNP9sSHIwo0Uc4DV+vzX!m;j(GYvHFBRLbeU)T(nv z(T@o^<&QoA#irvLE28i9O#!&_#>bF~FHimtaqYkg@Hcx1edrbj(1f6_gXNC!DG8v` zG{L@-5B~3d*U$?TLAe^3yDHpA9rJ5^V7<$OFLILoUp-|+7WHLSjpu!=)iP(Yehs#K zS{SZ4`vuN-YoZgZ)GDc{#4+o>rawHZazQj_omZaztXL+hb|G2GYeZ7{uuvmP_+-d zM^0i5cvf5n2}*V3V!C(=OPjGH5~lZL47AwQo>R`6Z=Z@7ftvFN2|!k*^~z_%U%jHu zf)9rPRzy$j|DdS-@BqNtEz#0IKFSQvBMbgp&V785(D9BWg{@2X~l}2@f__Ke__X5xlbiU@v3A1P!zv^cR0(D)mMAc3O>a` z9go=(Mg|A>nigqkX&b6qNEA{8&AOpG6v&R4L&Wb1eLRkdkC%vdO(j{m(4OT~KN+0~ z`URUo7WeU(+}S!V*Aa=pi)(xPQymSx@qczH5n!FMFI}z)LPPFm@2vHt5vP}Fq^AYr z(Z~Q-+{}vttt<0z5k#=Uzt&dgLjZ!o4IF$k!!xWQ$RH&ts5H?c?z^41W(VtN70$+O zKg6)#d-Lv1;j2ooD0-KFbd3VJke$_DVEo{+Dqv@e?J7$F{$fz=I?;s+UNg>{tNZ!$ zXTB8!M(-_E#3G;^$d(_esYtMvi0@5>hx6u=WQsYce>^}Dh?e8+6xk5h4Dvgsoo}^q z8_a0tug+YuHGbsvb^i^@tkNRV6H`G9akAk0Z}Uc%0~MO%7@utm$THl${h}4n;}S{4 z13O8Uik4CU0`CDc`L7B9GO8hHs(%OC4ClgiE5n1KfyLZ-5l0!(0D*@OAG)dkt5JR$ z%6O`5zAXEcly&*VD1fDnM>`O_z8GN|XHhN=*oq~iUmO+(wcmTwttj-CY8!Vk) z9D!+hb;cH6eQ$lF9AeOeX;8O-HbnXZSAHI!F~n;l&pv+F=K3alee&J=h(5n;?>=P% zJ7-w8y!$=xG@f4y@K+(2svkY-qdNAfP9}4_qK1iF%Pff2Og-Zjw$UwEXGM6s8wGV9 zbx2Y9szOvbcHB}R+6i0%P1D2EaA(A_}ClP&Av{Q5k@2&oyy<6D&qOr*cx9yV(+~O ze}Kq)sEzThn{|3Gs2AVIm222J1gAr|{h*vxy3I`@8#Ig|h=VTkX$J7&NF_A5IphvH zokMs;`^=r)QXhMb*3oPQRt%_S&{6vSok2*uJAYRx48k@=2aRpDIod(N^nCm+fYk98 zIBbwzWfEC9XmH)t61jQM-_BxCX8#KiGueq4Pe5ei_1yzsWq}%=nwc59eumU!zNz%= zw9YXTM+EnInKUCe2eXkI${a(E_sUxaoLP8<=%a{VR`aiVS{nzNt66bf99~(+b#V}% zI}Ani-u7Am)3m`QF~=o zYT?*(i{aJ;arqBkNkV07_2ryLp!uy~Q=19_G=@WM@U;LwH);^plcr5LAio}XX8LNf zBXXT`RwJ`*yCJk-N7DCBb{4M;2u@e4K_~yVuZhcHJfMQJ#WiB#KTN4@!FK?eO%=32KV#qR;`vYL7e`qmAAyO^1`ygGsilxg$kX9tK#W=EY4>9eV*cvnl+x8 zovuAY;K1t_{EKW8^$s5>w5m?Kr)_!c9Z9B?W0KO7$1Nk$=VAm`-zgVN#|IkP@Xb3_ z^%~x-r{S^q#-JYt|H=XHd@dw-@wGXpvxqGlZ z6zL{a7`hE{LB|MKM0rNm|Jdi*-|(9s*cb0bQs~J8z86^gn7b1G5__h6vIaT*m0cqO zqp%Hdit~U8*bYlHS`Q1@8o}IEUo4x!>63>l-@#L?8g7Gg{-)b%17i;+OPW`AXA7<^ z)@RBb6$#g}l8}Z3eb!m?z2&M%?vXB8p$C%BpxvEPQXE?JzX9}a;9u#)x3wDddOTpufGcGr>=F5#R}~NBaN(OeM-VE|SYV^8g=)`OO`jr) zNW48?NOQo>@@-ER5f~_yO__P6dKfj|&nsth-FWVM6-s6tYaAp?Ij}G!FshO1C@4a| zim$s3Es4(H4MvcWlV_!+HOoytgAzxaE=O`}y15;#trg(n*;enFrmKkxtL<3L#{B6y znxv+shJ_{#pL|odie-1m*6thk?7`~PY{ZEhF@4hY9L(?-M$e`deNA`#d^du-b$4iJ z$n#{{nFC6^VYUhh!BT)EBqk;Os+ z_Ey%``*(-!fKUtET|}pfp1gFR!N|~hWB1j_1-_ncXulBMUJ)r~wP)WtDT}qM7shcw zr#!l_naO&BcH^ZXLZBk)Z9&KqPnT z-N%m~85kJegF^Q?z_KSNC-YvvO7A1MnTc~umaL3+?eMF(c^W?Wr0UWWa60#iLzcMw z-7rd&n#ZEu6Qmu;S0skoP@|fKsk+Qs8a_nQqk*&2k_wsJ6!&$j#M&vDUlsK(#;2M%Khj#86bptGoX(FmmScDbi{BwC!Ros}!#ymN+BLEXt?|Ze4r(?JBdk z&k&eU13jppk9n~_JuH@~wqmqnRf*^Q#qcga8X{V-<$LX&5f|+gi&bj!=<6pF3L+7@ z6Ys3U;~8MO;M&LOCU~WL;=4z}<&mcptGhlb5c7N9jVm9sh|{W^R^amy75IqgwSxmw zGc#;a5@1Om04ow_kbxYp_z6>tbwkrpX(r0Mr~0;sps#14{T%5-D4U#?E@=s z;M_vj4ylSyLhH|b6WA)8Q2#0vAmU$y$GZn46BxV5aeUS0S1%K`YhmN8poPh!qoWTG z4~s*7T%>{yY4)_Mp~MC&40pz6WC9l;B|o=;y6t{pq17FsC8xh4SB~5o1gW zh5`opJc9eVl>?NRg_~Q84v+XV$hD$|7W-S-K8L2kw`LK84Kf(k_RQ*jwp|~6m@_(N z6NI(20JbU<`=%;1aJ9SghedkKpZS3HpOOaiQ_Y61vf)xJWLi5Y2{L+Z`EhWSgyxN3 zi@7Jc4d@PVc}T(&6cnF?S>gz89NQ(l&8*K^JDDIxy*LD(=@{m{n#O4X8iGul=e?4k zECJE}+@Oc<))!FI;A75W=Xt&wEjZd*)C*m2 zJn~K2xjmMfHtPE@KJX@>-Ky|ryO*=oUB-ux{gBM@UA z_dVA?%J!l#NlP&*I;FHdOkQ^CX`UaFA4*$3Fj z?V}39wdGL?dOrz_O!Es1w^yEC0YjDVmv=WtGkvGV$JH~9S65eML9&_&Z}vwiWr9~J zBf(YK?{30!D7UiK<}8ee_U05a%j+a3H?;A&RetH~5EFNo5I!i>*ZaPUS!we$Hm_H`#ifU z(i7KV!Kv|YP3OKqJk>0XUOl>2qAAbL?fFHbhB)Z#O8Df-tbbqTDxHiWV7;babP{S& zCc=kk8FI2ovC@zMFf-|F%9E=p(vL>#oLU|%lG@MsKdR<8(XNF)6U$|Pj?#H;=4{M=iWjpN%&WW%2iaM^6Cp6c0wpaDkgsGoNZr;~!9 zryUBoF`cpNLtf3+C=IVeG0T}KM^v1#Lkyz~L3U7s;eUs0#U^!H5a{clzATdtGJ@78 z(BF*5KEArcwp_^BI8S=_$2pSQUQ0kRgRke~)1KE}UKEUPN<5XX-m;Tr8qjZ^RJ?oS zH6~J5T<w8DX6RVjyCt>0@ubsi9Y+{3~=} zz+5O_52pjxSss|XHhSg$(>|f-rdoBJ3eUgih?4TbrU>$@>xqs*Mv#LD!;+v&9~ z!;DDod!g#aM%@*tAMNjwSs$GWCd*uDW>~Jf(9Ynh3}3_BqQEnAb8L5A-hlQ2!>7Dw zrdEH*Jst2`xT9YT{)w=S3R?_F^l|rdmRJW->G5^W%Hh86t07O*BoCPuHSG5WZcIH4 zjTUC=Z5?DXwbME=#ku}aFbP^(`t|#|H;h*{+~_jDDy|V!Lz5sqHKh{yDyuV9q`w>5 z($FxYHg|Gy;XP%z@Hva*JxDHvX_DUds1HCYtkT;#b-Jh%uSF|s9qbe!j2l+sZ`*o1kzKB=W8LDBQlB_yAN}! z4E9BdIE|eYJPZT5HWwwUdZ82!1y}3SIFPPSxMZ7(EZbErykAUSYQ`>1TprY2)5lZO z5A7$_w+Ti2(5`xOZ*!}i`JPZ?TT)P&lD$Rrokj2`P@i#2e;e)xo$X0}ICH|6?IOW` z5(`J+PF!jfn_Hgp6L4RLa_IqmK3 zJfD!qX#^d>Bg<#8%ly^C1`fvQzn>qm!>$MpV5KIL76b^svGLp-Jt0|DtRiuQMp{<3Kk`V}w} z<~(}VQUO!mEr5SD>rkHGi^Z_LS9yQ0n+54+Fg5jGnJ%OU%f7(IUg4`mCaCBniY4cx z&aoSw)&%wR1=G(y>w@+=;8})rbG-S~JjDXcR-1BiRRR7|^1+VAK>lSqZJG_5B+SA+ zaVI`c$RX?Xk_uqr596ah%FZj&ypmvM87DB=&tAYw(%u}?=eOsu0dnE4cgyqht%p$f zQjw>h!e{Oc8{PQfO=&WjQ>;`?I<+_cE`Loo53xxRLb>dKu$# zK(K#!ZGR@gLbS1umFP>zmaUhcJ-Le4lP7j`c-!1fWhgQb-terQc+l=FMQ=lZ^||cV zmjcCJhn{AbU;RY5{rA7gC};Y3Q<8{>udzGt81*%ZjqYQxQCPk6$;#x!u_ZI>+Sh6V zDiI)0aJ%Q_LsVRqW_BV|dbI42n0QbfXXR;`JQt`FHSh%{=Ezw^p{uv2zYGm8?m;^@ zqP?+EcC-Hy5dDz954k?G_X_wo?LRXf*!Rm}1nQSpl|eP#tk;=%9&Q#NhaynA*Riv7 z{AW#CGx)03h2h?STBRn;S%sA(9+)R{Ghv~llMYb(IXBnT)n&EcnZ44a11xIal|p(0E;W1KGT^A%n$(}eRkwK~0Lms71~J@X3G zfUm}2uR8|wHaD2>0sTfV^I};g$X|;^RcgTox~lFoRcAS<0V~1o?HmUytMZMJ%R;ZV zmn7sz^iXs>$kBJF$Q9|JK5Uv{K?T^^RX4lBf%zB1<=_&AxPI>7YUiM!AmirA#b^4+ zt@&E%{LaN&<4p9Xfg1ryOOxL^Ivo=W`Fmh;a(wu3f2gOYw>?Dma`Q)#=>2ir6eq0t zEk8fu!)CeJ9W(rTh^6{>Pf5v}zHh)#<^KNuO2DCr_Cwps-9Z=zAq_2U@`{91kS|qg zON#)zbV&EzAMB8Bp@Vm#Ns4uVT#HPtrI=n-FYigs;DW-_=DmFfx2B)7uL&EE8&|jF z1JVFVX71AJ5V8FjN`joLQ#Xy+7uj2@%HFToMwEIHooAm6Guz;YcY;?9sV6V*s;)}Q zePzzuax^2Y>TB!iyF1QKYQCCvLXO+?lwg!)J~gJVFgYDmoA?k?Yqy841Ca&y*8$Dn z#keD@+J!l15r?|ICmqpEW-7OX{OZH-kX!4R^Q=DzW4!0zaWTF(e(|Yp_labHm7tV- zpBTo~BWUDJ!j~T9Qj=h%sC*-_UYHu|0_j<(PB8|J_%s-h;Fk!mH{zI0l9YwMVHRIq1BAV=c33ccR`ehxOaeGKqRx5}>r z_KVW;>_Px0Rr4ZV9(?q@a~yMl2fsPdD)^JpE2>ltJ%gl;zbt^ZxOL;&`x{JuzDBkrucWpR1N%_CcK*I zL1UIULSp7rHSt|L{g=RS53#p{KeVMWV6~Ar&A5m=fJ`6Zgz1y@xmWuGGNWB7b^!`F zz)J?-&xk`BqUC1GYin!k75+Av;YAjTe0(ad@$TKbXHZ#&RQpStYv>gKtG;lZ>@)2Gr+8`M27yz4-xF? zWRWa=V&xJ~u$dUldoIweS(r=^j}enWU$ya>@ zw%V@bswOT7^RF~L_~Pg&m&H*HGG}9p2)X2y(M4RwR6v8a(F^ESo5VDhqf7-k{&Kj= z-t}+lgigB#fvpvBvqaL8ihdV#UAml%J!p}4_kyRSX$!B8#jab?en&;y`j{PW+y`{c zs(aMeYF}*To|E?#r5y&RYiK%p5bC+0yUmT;G|aFWSN{Ih_JtjunZ%AoXw(1j`@v2$ z%zsl3z#@*Ct+aaL9ynTY9Ee-vxB)m!Q(ynr$OtC#kZxqIzfKA6W`dgd22!fZSPvGq zJG0pEPntvuexlUf?)*vkVSsql#2d>L(2k#vQPQ)xr>K>c+ZB+(Dp=o2bFpk(OmS63 zS8APhn8F_s$-Ous9IN!Dwk`g(m1kL0ie&iINu{Zi@s(rG=z0FL6c8Z+fg&9qJ_GDo zIxi_3SoiNV7P8Dn8S7zVV?&Dqf4-VKI;}p&w}l+IzI)eQlGdA{KhO858F?gBhSl#R zhUVor$#HBms6r6~R1A_k8~ILABt-X@QV?VfMAtt4lGCjx#*? zH6Uk#M(7SATO0re4F`0mVWXM+0OECfcV`x`&#QUggTbM!qQZo4yEwLlyGc^MfNEUS zb#6APGM8$THL-G?=MDuAahTbNkA2KPJXxYZ<}&Xa^K}ZXs{L(Q`b+_b-8H|X+i^-O z8)`hfYgK*qp>xaip^gT0ePdSH?9s_VTSH4j1zg3g4%GEtuKKLZ<(%~C-!4iyPnmyw zuJa!G`*-|?&tCv`?o?se$;GR2{`&=!+RZ4omBI>%pYPwa)ca%$sBSY(!{>Z#gz41_ z_x6&@RRot;-)>ddnTZauR+~9OPVyJ*fCtMh}0}bP%@2)H`wH&`CG@(>7nw zb#B?(zIIE^hi6MiNtx5xDT*wR|0+fQyQOPhOM3;4%P_NI|wn)EP#8#Bi58OEb#!XgRLZSnFYus@)HJnl#|D87nO9Z#-E}#_WNY^b% zV{%BA@oWbb4bA(VvwDuB3n>QpzK_3CV!SdvofthZ7#PcwZvnGNDkPQ4xViRazedC52z76JUo8>{AqaH-)JLP z40SW-Q!tl68HFFjvbkPAFK&cB?K-Ot6|iR}++E16(w0bM$&7Cy7p~ zsi5KobBkNr*ZzkiM~|Q-PnUJ5o|!f(iCDYD-#U?LwS>}D?5=kZq0#(d_DpBLU1$3Q zX=k~R-6&7VbeSJ-gI+|xn)DFKI-3nMhjX9iO+6^dv2b8Ar@)dbs>c5;G{hYysI2p= zPE~$W{jv_{mX5-R6ubINGjcoT`u!`&a%OJsA82g`h~MB3QFwIaC&+Gf6-ND^h{T&?Ml(x&R>=Bp5{+Jg<{JCj-EJ8)iSpx}2zC zbd`0S1vunnW>1!pht=)u?=tf>(!AEw&iz?&!7cE(!{?Wgr!(bxgABk} zc3&(YjAONHD|(uqoc!7var*;1IDUBjz*Bu|WMJ#z%J^-~^nXRNb% zNt2t_?aHaqz}jVY@`udRhpx^}ulpWrfxnFZh$O`!FA>mUbmF*M)8NH!GNRU+GI zey6-JzrNOpY5&k_rz@fNikOKMyGMN*i33ccQ>AI zo2|UUmdlUU0<0W+vrhNuVbO~u$h99@e$4?AZTEev>PTsR`>xa|Yh0LpLw)mxlSV=N z>u>{wHG8tiej$c5+t0k#13eiIpfBeKVOm^cV$e2{R4D2o#gk)4+39oU>0&g2K{~9$ zYReYI71PlE%%n;70CHcaKjOHyUiD8SxS^m!R~Ibg#l(7WNca))UwkrXNS=(r246I2kq z#wN^-B7WarjJK&Yfz3yklv@s-7V!w~5SyYm-|(aebMB&8H_y+tUcl~)=aR9Pl86+T z>F%ZbO6N9gw*Ty?#8@)$T_w|CD+Hub0%+Aq$n5MaRXG}tEo~(C3_q9wSD_bfl3`*T z*IzhrlqexVVLa;E_2RP?1_U9q8QH%u#o1euo;hE&ckKScA8f7@x!1eg$_!s1)tH`C z@I=)$2*x(RK|gZ zGK#26sGxjfVirF0^SSvZt75tYOKGfp>ak=So(QJE!Xe4>%4TZ{P z?UqVRev+kpX+z>yKb}FMgu8w-hmA6xDwTj!xw9`3o}oP#1nWWlj?BIn8$cD10mHBK z@Fd5cGYM5ky;`*?h<4#99@W)olZAFBj9rUDx|!NqECLW=5~sOZ5L#VKxs zJsai9b==IRjb48G$}htq%Rq9@7)671$N_-H3z>b%Liu_KVQ%iJQpfep&D}4JkC45! zE^v`JL?si1f%+ut;ZYwHx@TRZ2vEJ!La@XhYCnQ(ZEU_s(^!hXp7JQ~P6PI~JEF2e zK{*}-WL=JG!D8p7YMedXK5~!=$~({Jr;T5}e9>fp0V0<|B(eU4j$Y9^aO8G&xFXa# z75L5zd$|pZ{5;R|GYi@;jzxMEeKzlhM&xRe<&wH>1zS$_b<$5F$G6jzI4Cr3%}q=z z26*mr%q$)qtPe&`4)<$kJI%IL_Xa(budS*FGh{i@T?9Pj(CmAG4^j##q#cpTfCaIX z>O@~}vb-Hc1Ur^6s9aAYAvctE&qq;pO2eP=@&MSi^uM#mUo1Dp_GCx^M#5AK&521ZXrrATdwg z5B6~X`78g8sQI4E=~kXrh-Og52hnwp!N5)1f+s8coJ?}z>T5pt!y_MF< z+%K=mr&g{x)=?RN!Xv(iWLGHK#Ev5C{J?SE%-@m(y8lUJIq_|$A6v>VD%aQ3ptS;F5>3`fX@%8P@rP%`Dc~D`Rzwpm;ud%y zY3CED3lIphN-8fe=i96!L>PNshcxv61I&&(L;Z+%P8e9Y5pWl{nTBQ!XrnMh2l|jg z$n9MKI0{{AQ`0=7OouhjiwuXdViruo^>`KfP3#2s?4>)lNKj(_U=e_`wF|Onr!EVw z6Rs@bMP!jl@zHl(;GIr*u5rr*$F1Cor`%fvyC}KGH8S*>0)E2tDH(Ch*RwiTmPAl7 zlu8<#U#1`=B;*zR3|QUH4j>YUzxgbT^MluykwZn- zI-0-9sLB(EM|>x!&>5?}Nrn*q#%!T>$GEwrBJJ81Q4~Qz(*ZkU`7 zw*_FFj6V#FjH%_5q^|7}JAqqLlzA&jGufanK$qu?!PeHcc6&Y5%*MAkDk=(jGwwaK z#BShYZ@(|_f+^UCzTq;?A?Bq`8fv{{pH9r6c}yN11nQCYmiaTL4Zofg76kBXqtu*; z@0ET^w}*=<=fnjT;4>cpt|n(1q|F2nHm+n%_?Lt)886*zP`Ma8uk(h~9Se5C`3H&& z7Xr?T&d>A(*ItG_pxD;FkovmlNWhsZ})f%?*7A_Y~h#JKCR?u{a(L1EEp4SFTvY2WujVLWE^9T6ogGGP< z%pQH8L2Y9OkWVpjF+6PTI-Zv09J9xu+ifY-ioBwDK>D(2mXX1DcA zv!}IAj(Kqn%n+K)p(-*xy1$n&Kn9EP9lP+%0L$AOk&Zb}ZZq6LA#N4dvUU%1drvF# z?`HM&m-n;EQ^Z|&=syr!Vu4NlXR*-@FHT?Hsp8(>4ySJV3lcv4Nw+UMpD9LTBt-K> zTh^$*{jB=VVDR!41E~X$@x~$CR&L1*mg9q&asJ*_=M8!Cr!e+yslq%P4-WN|ND&p( z;5zdN59#FL@89KqN}veD#Y~wfuRfbQdrhUZ{%m4>X|>k(>UI`H1rz+n1~jb(UT_Hw z8lDbiWEyCg7Kk#{CV)DImSjb_OIw8E_?hxFj`vhr59)YFt>YQ>SWJ&f(*7KmWku6X zevDQ{!8{MfwRKJz zB-)7!x?0&1MM5#-$HTvN$k?_!W^HV>H_(0<6mP%XkMX*?zf{uhscXvtqF40wYD2PT z-Id&mhD#lN_gzro9W98Y!sJE?Xanyh`pluz?BvLD_G?zt=7k@ zpH!(}eX%mNRmtbTe`PJ`}7k8BiL zJCddaChnt zR2Vbu(iSKV#z5-HSFfMWcX}t*H1PArnhwR8*Kj--nHjc!5&O6joUFlpA8wY^`&=!S z@pg`1yD+r>Yd8_XLxBDYB;&ITQ3G!hAbU$RlutL)L)q)uFrfmuAk?o| zP1I8r==6yI{rsEv*8n98=&AD{$cTyUogy>s*?MaKC5jFs(h;YVlX&EfxATtG_AXJl zae;q&o;m}7A{cy()cC^=J__Bz69itA>^SZt#e92r92atXdvzLJfZ9d{gE%tUj|*t( z=C%ZHuUqoSB!B5WEAt;x+8jOin%d`xi5*xRn0cMiq{{cUE#nHNHaOe(j`H;ntveB* z&A=t@eB!vm)aJw|HeOtNiV=F}DQjaTQ~Z$Xl(lWQlsNGu>vz*b%iscf)@sV9*W^Zc zy1A7A7I%ghp}}#VQMD}2*oR_?1(-4vx-EzuY6aSrXl;a-xyjq6n1!F+AS z*+`HbsD`_^e>QFx2{@FSSc3X#e6R~eN=FQ+j)%CA`O6D|7@FfUVN<4THVO_Uy_Y<6 z;ANoT>%_5rn?wy0GGT}^fnM3WTJ{QLIK>2A5^TCVS)~WWT#A=YHMeQgn~ks_&yTy z3Fr&}grS0h0zkC8x3~AN#6X>);VWd>U4TCHLa&Ii0dDQ*Zqb@9Hdvjt?PSfW*DO&O z{tOk8_hIjAdKF*kY)DkDOYSVgPFb-LDG(-vM8g_4Vzhq0n%FKgU`vJ)-@*CczWp9G z>!}1;b!actG@uI6)%w)xaE@YK`wWGphssZKa4{vVEZf*>V&)+Fnx34=S4%z&#e~9i z+ZdiTB{IR9woREuzL#!Sx&;HwI&T-M9q>B{Y&uXN*p<=QV{Sgytu6Oc{pp%`iJmg& z1us8!ehj4fejHFg!eOg)G;l0a>S*Y&BVHfFaN30(%*|jyd+ZlYwJk#e3qXTh#sJG! zTA%dA!x^}^eG9bX5GorJ&$2)Petuq_o_l$`OaJ<+K)0I+yzIqH{V$i7gDfpAB~Uuw zW=}8D0Ot$*nqf!)s@e}P8v&#Xphs4jnNG2RtM)k(z1+c)QVj#ldXbBqFuUiyVFQ2r z=m(mt0HR`|$WboPQ@gO5e9o3&#a7M)(H?e=?OO+^2{QW%p6tU|fmFEEWB7C0P1Bl- zMN#DtId_?QthrpZZ@FydDij8?JZ-|{db^|Eh`sgohTSIre}yDDcoyB+oCzFT9^Wy> z!_%Og+E(C&@rgxQk~yJIH+nAv^mlN((uc2(JL=?X1;E6?<^sQb@nYT!QtE>2d-MUV ztj_>%3YP;}!~m_W;NW23d9jM40)RjWm?4rHDQ<|nSy53zPVX|y0zSx&FBGk&WSuhh z@YIzR0hy$?9rj!X6ek~l^LLu4QMm^R%7i~4R_i!Izw@neBPW%b*4+%>u(Pf_p-fLFx7Iox zbs_4*oibngLecmRs9&7x`4c3fpvB9_xAUbDz(V=4hO#+yHT)nZQvteb<$72uY#wc& z_GvL_BAvdvU#|YE$MEq1qOPv@QpW$T8esEnuC48{=TabB4;{e6c_k%RSALDYngFd4 zz#owO$z11u`GkSYlzhbLbPthBV+FU0$0)1@%^2UW3|MyngP|L`LwFTj)G%%sV*!tt zf{Eu&Du$D3wnm63?5dVh?Bfm82D4Qw>?OWR zXBU*^b6dE+_Loe4|H|L$0>%-=AF437pVamqNg7N|PToHr4FA8DyL<5r2f5ZPoJ|OX zj8+W^6aXnOCP1r1oqr6L4cWwc`SPFxc?oKy37jjbBk5?9*S>$ZC8Tg&c!S+iJwOyi zo9tQeJEQ#`?{hy(Vg+YP%VB(5|Kf6cM?m{iXLLH3DAZVLDfW(9vr*9c*=L+t+|7H+Jssnv9j( zDacX-^Axy7-$SJ!KYMz9B{l=(QDl(`FsYvxBumpIy!_`V7@{8B0#147q+{8u;GBIc zK%Gk_zZRHxkB#28srxCKKJu%`GQdibtt&|qJocq6%f`_Qc~>T7y-2*XJURd5r>;!x z*H9@ciSYGOzQ@&aot}w5gqNve_b(x=P)m0S98DKBSFB36 z=?!K~WuMvz(60J_-?%(6^uZL~VIknDT1C+tCSn;i52^423}7|!e+?D5+(IB|dIvCl z06+;qn-=v+O&o}@k$wo}2VH`(o{R+hl^k0FMJb7TbNU1 z7s2(3!pAf zy~iZf^&ZgVp7R|0o%lQNM|jQtj&c(xjD-du=jPKO`~5#}G5+UoAR_?Mq$Q~04=dNZGUrTr|Fa)sEcNm)(W1dGFS7M5U8ZN=fBGL*7rvlTX?Cm8dNU7rEt|z zV`qP*Fq^n@tMl@CWVZ!t?I+fXNv1+kob(e0D%d;ZW|dbY>9P!`Ebs}(Db?=ugB1+- z)7*n{Afi?j=^a2G1^Bm5)>G@c$6tU>2_)vz|7U*xzU@B`26Tu%i2GB+(qqh-N#n|o zGmHo(!&4OLOK+)$43fc<8@LCv&6TLN7#F@QZ64aQn5=c_X_WB%exn7m0^^d&)lvIc zLqHAs%e{=820zun&wioC7Y;<3sN8qHS~iv-AQ~b!qJbJ#^Saio<;S44F(PFmH1*jl z_6mbOr$1f9!o(jdc3vHP1m$;Fbp^!Wd-^$yz5#%II$%o=ZvOwa)HH&doGpw;Ky31@ zHjW%Unl4|RvtDQT6AZDaia@zVWVjVV8um19h>7+}+D1cp?!xzjA$ zzH_no_&ehJs)Y{kbCF01tyQC75cYcs-2v>ng{8qpn=a)E zm`@U!`E`zU{riW3<3YY-2Kalx$hV_Ft8*rWnvok)YV6Nx${XoS>2NjcQMrS9deP@s zu@JxVg0=-gv*`AIDm5QxYwf^*Tb8Wku*YJl>NA3`(jT5WsN36;xZt;Jfieqf3G_C< zTOvI?(#V1AoVous-K1ArfvT}x(l{rmq2h$BaCaBr30nDNfE?8wM*C_%ZA>^PojMK~ z8GYWcO_c_$Dju+^wl1Vo`+uy84mO~jVBWlCx}Q4|3qN0wARIUwbl!&LG+Q+`EbQ@pfawn|Ipm8^{h}R+k zXrdUQptz|^iL1)v8s6-HZEyLOpjUHPU6bW37=Qtmol1#%I0U^*3w#5l5&yED3L|7#M&5ukGJwNi$Y+l$>T2B&|2fYJ&P z*jy;y|iM?zGMb4_gxCqzxA%6--tcBH-0w z@26v4L;MGvzI`>bm7g$PXyFnNXsF?)t*Ec;Z((CkAMz-I6dApMIVj}t1@O~H`j54jvaZ6R2+sQNwX|Uy*p#b1 zw&v(ae+W!~04TGo7YeZGi_zQv`G!FCO2EK>0Rxv&G)IonF`$5F(qxlwjO3d7veVh1 z2JCmZo7b@lw)|#T9fIy>c5d2VOd~vu8APEmMzZ1%B~q)+fVF`t(my^bW^G_nBYWq$ z&YZ6eSr`Y(n!3bEAlIGjE82JcZFI07HNKc^6!65JXDNKmZ>f0xrHwexq1wapyv-2| zau6lB`%Pt4#19!BUc_Dx12FXR0t%n52>HkhZ3!tOXJAdDg>AE5KlZFcLpW(}0L_Vk z2oHT_LGxx?SeDQEi|4V5;M14w{&xg1K0zq?0R6X|&t}*xrr)lsv zp93Qb5&vkX+C~riErKWD<#b1b2^G7rypDtQ9u^ z0QZ&zh<=0|CiH&HzRs3sMK6LMk{$WcOIeF21?PN}EvB@p_p(F=!CA&ZM9M8iGHy57 zPfKwKGQTb4=l}dux9X#cw0@^2(DW7F7+3o&v20|$MvTh!v>|TxLzJDxEC`xGr4=dt z0{NLhza=X8k>$__P-X^}TJ`t!3sOAqpP#HPbSJV#^C-nR+PKo{lm{0pl)9@qDGPwgXy1ii@dNlWedy>`yv^RpUSPa}|)10Wph z;z)y*PipNW1rCeTk+{@mf_%&FrD>jmL5UhxluOqq=Eqcq_;odKoyNuBkMQ8M@5&?1 z9aq5N1@36S^hjY!Pce)9*xKE;hf1ex6RuFDe@mL)G>6Eu z@|}hB6oJuA?cWxEJ^YRETfOL5@EH{e46Xa(OBVg_g}f%DQQw?jkV}>?j%OvI6VrRE9{3y>7<~^#* z&3l@IA;JLMwq_i1!kIV)Sb*KW+xayc220!faZs%ABgg{JD~e3>-K|)@bg{s_b;3Sp zk|I)$3a8$EX{dFXygOdzhbBDsiVvXtAIgL5a;t%3i#tONfR52hlmFhQSk+UXK$C1V znIw$;opVgekGMZfOjig=vTP3b{O&f$9feYV4_NWMECYw!ppwM&zpUCq+-o zBn*E@at%KTjXw)yU;g^lK(_LnFdRc8d;*=p=Bw!Okz8@$O3!s=tvb z5SEGmH0LdwmI7pODN{>xY>UxU2`4C8iOtjfw8MLqPtb?|u=~rksFxZ?Uebwrzxp$c z(c%vpm!>ji1dF+?3uwqd!euH!p)ge!%PAUlAU;5AZ$WTg5i8SVX1Gs;G^2DGT;TEz zK7FjL|2?)xlzF`4#RjXs+~bJ4?23}InD+yuO$YYGOch*l9&Xw%*umbXSh32~tVFv< z3A9WCuWw;YEJlp$ypN>vH0=)*hVG;+c>k__i$O7;TJUbGFN!L_G2fbxSy(Kh9`%DL z$a_i34oW#blf~HNFfMZ2}OzIdzQBh{0l{AT*)S(tNvfZZwqD?zt|wOp>XY zn-22W@#iq@EII)YK;dMzXHleck}+F!%;?+nB{@T0z`1k#>#I+=Z!G#HV=*(@nCp9k zvsz(Rue7;=%z{%W7^1O%GDA8uN*isXR3=1Z-;eG-vyrmlqmtC{W><)TQr|F%Kz`QJ zQ((M=nG$i9?h)OK?P!Q<#S*S49Xh(iDBnp^EW|`-M!@4g+Xx$Cv0@g346ry#OMP3U zWYnYV`c33~0!j+bNGGq}O>1^FAGO>0#aVZGX3+Q zh|5+2c&Qt%$_QGKpuXp`YXQCYsdkDfWblE!h3Z^;Je!^|qZf(~k_h344{w_+fo)3-Uf>j1>{FI&ZU$n#%4 zKZb3`Oum1wZ3EvdC(z1SISr3Q(?u5jtQvM`iXa$Vw`0{|r#3IVlm`&z|gxwN9+ z>t%}>g!)(ej|(Fs#sobs$Re5glELVaY9RzE;`tt)raYs8d%qQB(XhB4Pi=|WDUTP@ zp|m;kMYq?=25$X4QHmq^K@acs;sMJ#Q;2wD>a+##vPb$c-hEpbVJc;ON3zttHIz5O z;HZ?h(DreqS_JHUj+q-5c2(gcjyf8m5#qAsC$9RK>8-E^Ba|3xqwh(!?U<2pt%%>r1ek3Vrrs()G>rL?QMrh)U*Gd4dHruUkUQw6 z%{&c`(ZIDGajsr!u4G^O5wq|Yk6Us3RmTFJE!r)8OeS!Z?QzF52Qn`^KWH>V2WP?` z^^>pvjfftbROjXqRQasPVe)OPu*PU>op1y3Cp@}9Zd@5MmenQyeO2~1V=@}*jK*At zuxkFx$hwZYV)!xsz<3A|TW$i~RR&Oz@?EGAu_nHt9{ZVDwbCA{ls!b+<#!`WTl~FE z4TnM<Pyj6=19WvRr?>C5i4J$FTN0@rFP*TI%cE-#?fBJTB{sYkIt3U4NVRBZ!2` zTb$Ww=@~4OsWr4lFV6)bu+5^_!1L)wp9}TQm5;PZ#OJ5Oncfl*h0VyN*3(C>lH|I< zo?lb?t*47A1&|ad%qKOP3SPm#=`*5%dE>g>3t)s271&XyTs^=ex!UVI`8xZR6_^vQ zqU>JWuuE%H?)wM=#y~#-+Z}KU8Zjuc0?5<9r-?|xeJMJ|n}|Ph5sXrH_&=O8B2Txu zZHd|PgOZb!-ntc+>vdN}oWB3qDN|@ijCD6YPu1zDtC8MX`FaNPIr6QP36$JftBOY3&n8V~Ko3FLh{19ji+Zp1 zj$l*Ie4dLw6PG<)4e#s_?cNjA;ifC7Yg74 z8lbffRvIi5oRV6^pEE2fVm`wTJ`^^niA(F^#!2Mx;@dv_*7v&A7Jy~Y6^-MAtL$;D zQm5q@tnwv-&wk-D&Hx;d)LGBLQ_-BGml5iXb(zp19fM&J- zhpVp+h`Q^xeiWq}0cmNZo1v8ukVd3ax*0kRx>I^60TJnLkWPp09=fGL1iynm@4ff^ z=062y=6B9MyVlz49H4Lx)*_p;kCg#-lm~COGRz8}m^^_d{DY{=*4|b4~_Dz7Y^ZBw+UEdRTa{`KC0;y ze8Q_F|F$7?1-`6UP@m915AV|^Z(1fRrj?1tXD!5nD)lo9=cc!>G1Yv9_D+tZD~T!u zz-0w>0Q9cCjvj!C&|4BLs37b2I-Lf9{h%yb986!y{;g|cd^H!8JEa-dBF;-13HlbX zQYt9!nk<4*dETj=_BL(`Pb4$ws|dT?6Kp_=<3a6wSzKD$GvV=})bc)&_L&q-LfHX! z5z`Km;RtV0qi>vRZKk8~{=H?L__8k*#{5ccB$3&io+IQ&S6v2>aONNTd`=yb|{@$8I!4;Cv8wHvGU8D_s-rUK9o4iPN(f zh%e7NQf=lS)e_`jFF4@*2YT*FRKR}4ZNjAhTKj5?yEn8A5U-_Cqzdu4?dx3 z6=nrR)-LUYO^?-J&P&(mUXINI!V&-C2RS{k9t5b4lxg~AtNQ{%!l>~{Xj+Pf_sAT? z3#3SwbI!OZ) zihGn{O&(XwJ90`-GhHMX(r)KQEynHIxj?9}!$rUthl0KJ>*eX#f{+pO(`>xLZ|@U} zVMV+#Ra?Y^G9(>ObMfpuzhNpJA;u+Db}D1yK;$zb6gXa{sH#*ZX}xvrOzEyz)5yn~ zVAMoeD%7obJ?>KA{kGBu5%EHy#$z@8I+Y3!^y^V%`dgN34@6Wvah_5BX>#mVx3pwT zJi13oTgX-|rqTe{c$X6>jt>6$eWt=vfpchd&H*ZIn>LL$oH&uFFAiUPT31C&_?c5f z*ETX=wxS#SJWB?Yl9i7ei}Ov;eMn4b;if4qAcgz-z`FOB-Arn=isR1v1#)eMLd|Cvhvi66VjGcGXkCJ^rE7N zm8<<27qM@xC1J66YOv~h{wF5oE&0z!GL$7_Uh~=OSsuPe6^ZB{*osY99}h$-Bq@j8 zosXS1-HF-vmYC2@PW|GX?T$UzvuwCcHjqY*&Q!rw` z_VyX`Mh#1T6g9DWZKq(Gj`yqjOuD;v{apdwSY5?W)#ZxR8k+~$2qLvKre_q%3Df9G zwBC`Lf}J~arQR+hBBXU}y8BiJty&fE#W`g8iNg<+ zNoF`LtG>czsfMk%j~#M1HCGjh*$w!gML1j85hw6@$eWe~JCyNvlI#b{M^OrE%n9{} z++(ae2@Muce;!yh_$|xS)PAY*i%sTqn6QZ)gg@#%L+27AmW`G{h`m~c$lLq+wkB!y zC)>=J-?tx~cdJO)xpx^lnyS>mO5o&^m8o5v4|9gSeU!% zd5-iBXqpwM&indp{miWeqt(5`sPXL$QD{HWKF}s0$<%g|ko>fcZ`veh?(y=`!Aqy4 z_?vqz;Il^o==A+XuMkF|BP`j%_g1VxD)sT=KP9we#k{3G58nQBfU@zxhWmUxqPAFEGtQROJN{QW%Mql4f4w5mm6^6*T z)n6~G%Bowx_2d!CDN8=(*))A#1h;QNegel@Xvat@iIKH)&&--14O-iC^0rVW&9Qj` zZ>oWE-~SSPwB6M;kFCCMW%P)Ysf#v17)M2r1jb`AghxY;pB@&mBv?H&`*psE|D=eLiK8iZTiX_V=G;k3XD*)bYM4)Y|D%J% zP&&SasYZ>`8j&wiV$5s3qzuDI`})dFcFv5Cdr^>_;(#YPl9vy=vb>+>WX&=@@%!e> zJI?#g{piOlpX|NN+%@$q{ow`gz2p=npcR$0RY;x^6KTJZyyo*0qYwwtgqOi__;u!J z^@}gDduya77B0ZfRK^4C0Xl|1sAtZ&OC6!}y!OC5rlz4QaTLZ`X+4_I zz)T&)F}m*cbM-;t)FaY%99iy9?c}v@U9axqeT%DL%~}z{oCIpt_^W%Y9D9MMq^#ca zX>(Y0Gs&Smt|zBBH3o*Uxr>waOufXaM==!yZe^wMqSxRH6?c zfv4r;X9J7xB4ww6eR_=X+MNdUV6(&%2aG?_o>|@tpW=Vv_ZiJagUJ6$ya+yXHJ)oD zpNdXiV3#?k+aS}IMG9!i8qR!+MRtk!mi>Hacyc7U%KEQgG6-etJ zCeAoEa7!az6Eq?T>oWSJ_qWCKzQs5zAfd?MDW7vNSAC=-f=V}a`CX_b=ab27r)9W{ zzK*ySk(xaY0qjWEShxLn)-UwtAMA5UV28OWJkKt}7HoWS9{CbAX? zGAnHdlETgs6)P%tM6ZN+SE*c^xG9UF3s`76uHyzb~DZph9k%Vs^OwKLt}tU-m2T0{fOtey zZ#UWtP%uEJFUPVhD_&=~{bFrUn_WHxA}QQ1?)9E1qZHlLJ6mv7k~fAbBNFWNEhdLz z9w9BPWVu?zog^|sh7A?#%&=BRkakHO$L;z#(Mk1DUr9$Mf8)lROcKCPMzJf2y7XwZ-Q!Ze;&W5uk!GWF)5HtY=JZLt_Fs{>EBoV7u@menpsHy#q4pZ9cy77JA)uqI(G996W<#ogovtH3ba2CPv z_!t8}`{V_z?enGBhQ-{$ijR9vP6XDEX|PKf*-FvR@*6y*l-==2zn%WFb|HSvL~krm z78}fRwf_BYV}l_N2zD0)j?}0cyjHdcx)gxeM{{Up8_9SaKwFq0rCu+k-VmTEtL}PZ?rSI*VwAg8^0aNra~k= z7QoSL-WLRFF&Q;QTsU(QICJfQ`qoFB9*)|g58_v8^l13Lm+@|;wqbjnsmcZ@_Q@S1SOit zlKNW}dwB&w(Ss#RT%8qA_c4d)a-PL^(DWCVy+a6bAYX-J{~Zf&%(vI66WF^7+( z?nnTbgJ1Rz{DUj#_$RS>HP~0rV(9Lg2bg0de{e5AKajh+y0T9LccQ-^`R7gXWIw~ zhOcf=SepFq9G$%l4Sr|j@kU9m`9mX^Z~LQ&MPvzujw~%LDawJRdQ<+m8|BY``&!!B zvGMY%d)@+Ckb%B_;E~|+{~=|7Ppx@L6AO5Vbp@&h29%h1e;!eY*nmxC;a3_AhFOS8 z>G)@H?jY{FZ;7k>T192~vub2yB ztkG;|0~y1iO!41$Iv#fl3e3Ofz}2Tzzb|{Xx3`(e@Is})mE1l+-e3IZNc5+Ej;0^& z7cCdfmzN&;t`()FLy8#Snu4~6qGkX6z~_P;Kv=I_Crjw+`nrSJ@9r76-zq9A=b+&! zh8zAIO;XAbu`)Mjh?Q0Gw6WO*WMwmv{kNALRNb!_C&5tQDPY9e^Iy@C#gyARTx|eg+pQfPpO_i4SBZ4By?j>H#H`sFGhNnZL;i#ek&$ zP=+X=Q$ME%R(E$zxuY471aCfd1bclasd7cEfhY3+JvF~z2Yx6WKmVj%Ff$X=`vH`@ zu0u9HuQ&6d>(2Po zs_g_#18RDQ)*F6iLC@n{r}dH&_IZyNFS^;izN}k+*co3~z|y>Wh89xUr=|4t_wY(b zZxjWO|HyRgLm7tE>_&@Q)E6DHtgTIb-~gVcRLVq=3qp_8WnGwYExrQ_1M8WEw1OEO z73JmSb-YZZBG8(i}^eV`V$1rI{YRJqkCJngC*{ zax0*P4OXJ{t6r!88yl8DBgSRnjr9jo4Ln#zMi7RtVUc`X!6lXD~jDzYa`c ziR4A_^ayz5M`UDR8~yIA@Jm%)9fXI6fSf$ymF9QbmkYFNG z@Ih*Y>Wwm3Y~k;#5gQj2#YA6}sHv&T!j{|c?y5;4fs^29sPZpKM1*w+C`_Y;6XW!; zJ7<#Fp73HD0$3$VYfh-~`axGdkZPS|{p$?5p%xzQg%8{Rp2Ih>>YqzK@c}0g4Q`^R z`Z*l@18H{Y_PiDdRnAtC?Ggg%=o)x`4&N@r%0?m zw*;=O?(37ovU`~~dVO)Y`t$CJ3?(EOaKHMD?spJ; zfY?0!_i4;5>hkis6sL|)Pr=^r<# z@WYFmT5x;^S&OR6o$Gzhg5uK-yMH>UKJ{QJ^(FUJkmc~STm?wBE9iQTN+^aup%EJZ zAT``~Ag%ULxZ!7jRDR;r8M8Y;g79jCRt-#Ep|ck6vFq0pJ&LWm`D6iO&@8pMV{R0+ zf%+0u2acIml6DBmkCSfM(ExxFyN6eMs6F3eV}H08o7nSu>g`>rSBSGCZcQHfOu^}` zNv&gkXMMQ^Lt)58XUFf1`UyZeP_+S>AmFNaoPI1WtC;8zm`23zpQX8%0MP7qb)U*W zV@{EIye0}URu90X12ocmdTQzuLClYw=Z<>HrZLygX%XB<8ZJ-t)Vg=u=qP*Gko%_iY;va;Af{2rrtpSJS z(nP_Da)yPOD;Qeoc@@0SU=t;@yo~7UeW0l8B}N>HxieHOj5Ev@d9e0Mt7d#Z z>HeAaY%a_^x$1HqcLOd^mvq+9mGMfyD~^nMBk}i zJu~l7mSQS~x&w+zQ}t(QBfDJ%X75X68%PqM-N@TqOESA&I$FArnotYlJISk|jX60U zsq3iP{m72=clr)2iqRqoh~;?NtD0SA`%1IcP*ihG?z-lv(<=ETW=Q!U6h+^Eg-yrVm` zh;%S|RN%SPuCr+vM!=}NQv3x6bWCz98kJ)<#-0}MH#iSTtkXN5U`s8L&akOZ6&}1C z8G8I4QuyeO<@?}%%4eRYs;WEvKY?bne|?2FqjoNb&kBYm@f!bFAUMl+dNiJ?EV!s{ zvjnlSk0>ciTJHicujjrnl>Lh(_2b>k(D;kN;)D502&nkk>pixaDWv5?DuW$36>}bU zrj!$QRR5?`2)m(d0kHD|N_6js7`^8E>i8BfYyiv?m_SkBxt9#Q3=7zTF$;LCgit0D zHQa-va+Ps-+pxhYQcTZ9jdI}@(6Z3KcXg_yD17V+4NzWJ}A68ERKtUf@ zm1_X1%k8mgym?XZ@-p0I=agie1pmMn_#!Z$8P)P=Eh~a7;W^CpIPXdGo2t(#keo3t z9$_Ws9Ioe@J8K1Fxodz*195AVxM5RQTR{8Y)AAtQxImqhn0WKE1tAepyV*k!SYx0{ z6+l2Hb3xt4rm$cC?OCB=w*GKyZY~)3kvIH_&n$J~;O2II;1!pWQgC=Bz{f`jw#XQ4 zQOosmtjNd*#!|(^q3jCd_gSxPgo(`%5%(s+VW}H$_X{H;FL=33x@f(-mH1IRs3uX} z-6<^ym9^2qSo)I`XaZ2^`=UAxu-K?G!0Y%>h=)M8Y@8JSKASxz{uwU_xuzsSLNBJR zc&6_8LrjXa$7o{AOxf2n8y!P~wvTgZ>Up{7_H)Lv(`FMPIZy^Ej-0HZj~0YSsPShQ zVQqnyTe8#?QM1!*bVp76<9wqrJ)TpYB!>6F=HHiCtq9v4Hv4Kv@tx@lLq0iW(#$Yh zr}sp)@;p!N8(7zGlS74NL_^6nRihj%&mIBE

0a2@5{H>?48QyPPPJb$Q-Mf7UJ z4CZX$upjR1FnhqWC)Cg0Pj|K)UbkRj?ta7UXgR#!88uDv?YIBI;L6b=Cnjf+Q0vgu zR>wTJ?CN3xi1E%J3Oh$_K;WRE(mROIkaHD?!f%n_Y=p{txII}~uB+&Nu8|Q(D{kue zSinjWrg^FYG=xMSO-&tUfBRX)w`HLI$>TOVX50osaca*m;)E&r`P=1XFvC`XJ)40Z zc9Y+u9&K-c7R1WZjbiIzN)A{JSZU)}zI*rEjEsbwJeNhDCRR3=4blf@sVXkj!PJYj zyC;aQ#)UwVX!6~9f@anlnD(B8Nj>)mSy^Jb5J;H$hFT2_3oH_8@m!dX5;G{BrPlE+VygF>E4mt==&c5M8M)0Z1}2b^q&@i}-DqzAy#_c= zZCTiJgAXIKuqZ>z_C??8HbA@iE{u*)jUW3gf$;eea}QyTnLNNxA?AdMF>-VDwLXC} z6$LSF7V7L}Kz}S{;9}Jp0lwFf5gJa}j`9+NEG4@!e%qDuv!8=fpSoOP!g0*7VIC3>y(ZV21m zWw@}EM-%Fyq(+?Pte^gRP!n#(LPe)%9?nC23(6|}=-~T6>8x9%s`+DfV^v^vvm;AY z#0sBQm^2N%kOKf14?T;^ zQIV15$NG(K&%ppt-}vaL94iL+qwlkN2wSJwN3Yfd>#t8R+vp{P=-^G9*dlcsTRQ^- zn?d4>n(|0(oN(LJIv5yIYm(6LGN_ncJZWKsdV*mijJeVH2)ZGd2@ihLn|Kn! zBg7yXs&k(*N#lzzxrKshzOD1c0ng_oTZ#o*0%wogCJb0Z2?uQ}Gb{ht1k3dC}bwSxO<5!yOd9iLsEL&&(*lW7<~eA;K8>R+P^Vl zk>R-k>Mc(o8QTiRuq)$0jtx?ljMro&B%VLF^To?y);2a%i?9D(qCvGqwnB`g?;f2q zBi}WV^i~Cf&56`aUWo`O^RDHyqERM6g1gME-@PEYdQCLxHa~BpkUXi(25mVcYhi_A zN5A?xL8(pN{)RfNDp$!Hp*IQH+MwviiuU@2=|;qK1pLl@H)sM69+ox9CYSUYPy64pAjyyEiJ!SV&1~ z1B{4eYNQ%@J)P`Xbbfbz?Z7!VxD;%P=C6r(Nf3AZJy%t$j5EWlO!qG_Z*_0XCYRO~Cf8AqrJ$(C2#q?!Kkv3bb9v(Ev{>kQR z@6&N06(7p;88JPE#Z_Kb<9vjP9;(E_>JTi>pA0e4#1e$oPIUKcB|=J1v!5X%vTN@=_?)m< zv+RI3rhjyBPx!b-Sy>G+zB4#B;Hm!m*mybd3JEbMyp*O~tH~tkHeaS`D8e`lD^<+T8|}N~D{9>Urfl;pH{BfzQ!tCKDhrS8DWg!DxW}@DTN<+G1I1ZA7a=tJc7@ot8}-jGvdhCXf2shuz4Rbk>mjroGoUH$-v0Q^Nz<-Ur>85;Uz28Z z;Zo>K0aIVHxsk+p7IEfw?9lF_H-MSeUJM2wP>3DMrpNvG@HZ=&y)0dO zxS6XT^^UCW(HcDDJ|dCIS$-jKjp#7g^JotsLv}hgovSGo?k^x-smoyG%UwO z5hlO;lhfycp3U3S=t2Cw{Xed)>ucxz2zLR3n+9%! z_n!^=_aW_yQ)W_HVoNp_zxJa@k7}qIzp^%vjH?q9%V%eAWan?>H@8VOtP9?_nqutq z9dM&WCJG`(CJ7=%CJ&-OZY`)qXYU*=@;m6Gcm^BmTt}7TIWF{onsbpl>DYIXg79j~ z@VU1G_ukhpczlof5E+fF!$|7#2-d<~X^jE@%D8@|Qwn~5!Ac>-UDe)!jzcAw_-eGb znCelA-FChs|%y`xZ>UeA?7=*Gb2=a7TnHGerf zZyGd!znLKR2v|*_2b0Egm~)<@_c=4|D`D>kP6C9QuGx|TBD3KXwV@D#Ke-5gXb2C8 z3$bn`-9X!T&dEf+S)@zgU{i0&ZDY%#*z?x)w7HsMXh713_Vf6w$wVH0J_+~jb9%Uu z20!*o&Hm==`6fwx9(jE3u*3RmDnawkeh&TS^7AjG(faR8X**mUk3tpMWc%Y9JcB7A z9CkA}HXz3^i9i`+9ck~D^R1ibCTn(Y8e)NyG znd9{Em^l7dX~y+gp*m+tP7Zn4%GNaY>Gug92$y`Swh9GlJoM-MLyfnHHLJzCeR0QO z7EV$)DxBqs2uTO~kWAe0lro&X%kkUR8_dSK>wuqu49`|i>}Ro%Adfir+>|KGG+mrB z&L|k|e^*C*{csc&O(B3CrVuFk?OS`Y(W!Bw8J-sli0;qVOkOZH-~Z(6zluIE=PC^| zR>z?fikQJLrn5U}FsQTb=Duf5_1Yv4=YPz0)060sBtmm7D8Ww4)?CM45c9V0l?AkJ zxdj;P-_uC&gQd|&>eo@Z#MmpJ>GwP-))hqWBc zR}zu5^IL+~dYzDUqnOKytJWTnlCT{S991TfO z>oULVz&RzkpUh}vgrbK(vM+n_ZUP!iwhhCuZRo0n#oI+f> z`@3ToA|WaQ4#HdK*;*J-gKf!jAw-c6PHaqv>$!!%PF|sahX5I##a|l$>vET|@=VO@ zHZSO>Wj;H>U9#LN$ZB*K#@`IAa?L9q>GL!$PuccN0Ev~WNK6wJ_9MK^^jD)H+9NJz z$3Jg*p`MlsFI?2@m7qxG^8VtZPAq;juL!uq%v^0mg)OP+Ww2%Xn2YcjOve_17mBOM zr5@P^-ruq;cO%>DpPzLnj#e}T&K2~oP9g=}Z4tkiAq90+H1?Md3E%oMCGsDkKHc>( zf9@(J;bv>SLK=MkApWDR;HfwRy}0INgeF0Wgp->U2;vp848#%9xfMK`vsL4LCBfQ@ z6`J**g2g`IJQO&WePfO`U+Xycjc-=y_Z9BHj_A%R=thOUnq-Zbcn8@sob+-z?3dZ2X6j=@v*B%tK^b9Rl|q|D zm1k4+=i{DQ!DD#yiDQfXBqZLS_HAi|oN@_JYwz*4ev`cD(uc&qK)I2 z1jN>UqP=T?ZFd!dTp!kI>bAU09ic~_rt=I*x)c1?-C3y#`%D-gx8YK%C$A%EZ+5~_ zR^am_B|R-qIPvbM1r0&7mO^tRlU&2H7d&`it*~mEMiY59+27_Wz39%TA#kz`3HjIG zgNS(H=&cef%0^Ke-qkCz)zkE+MoQ2id`U0^FIx+H){yzR57{#T%~y~B^h zv+;{`w6Ahxq5(>AoDQx~+0M@lItIANiTGt34g!G~=jRp`^*sZ^dW#;bJegOR>bLWo zVi!QZE+{atxJ16#x4O*Wlk}<^@6s<_Boon7tvf^GPxYa~NA55D`t|nq@OK8A|D6J4c^+$gb~bfsEV$@V>!6~z zOyahkxaEBU_-dI$@DkRh!ylLL-7^T;h-*s&U&20HeBN3CrjWbOebqnbY;`;))8GI%{A3z@Ury}G9@|ERh%(a}<)E|m9fv8;l`q% zceSsrnQ91%h`^>U!QGqk=ix^CqQyEttu3?b2>fKj95ADCz$V_=`kP}#VC5+`jM}jI z@C@+&rr`B&DbW|ccz1R7yu_G>G$o$od7c^ z+DK~HZ{|(z#C`%4TQ9pfvL0wuwl`x_ilC+~dhN2YUY(V`!=gvQCdbt%DJ`{#?CI&T z8Otm0lc{<+&qbu-Xkl+3Mag^T84&RtOj3wvW0UYyKf7)1|NkQarjv=g<=%QtVf&_> z7Fe`V&gR#0=!S`fBYMRrHhM;qvP?2gqM|ynQd$y{H=sf_7*L4=yAB>jPGgEK|9{^97&y4Q_Xob@?)_~Xu8wP> zPfwHEmRJ@tf*_bqAp|L5WgtM3uYSLBPt5&6Br1 zyB?M2KtodG2IzUy*`%&**L9EhEwsSedOhh~nntK_AO672HXiL=dr<5OPOv$j#%pos zfq!-4TX1UIaa6wM)t%)0cFtD33rw@-RcK|A(pAJaEam?M5SN-QX-^N2vWLY$}86+3)Vdigdwy|2<_5 zdZ&IZw`Zjt?3@lzjJdhFJGuJPlAXOhJu7ANO>?Wzx(?P)oq>PXtUsOuT-Ia_U@`(I zJf83RL66P?lv~im>nb(yZ4cNF=rXS;4o#tN zQk>~JQ#!cS9AsU=1(ennk1kLkS(rcPE7ehWDP}`WuZvBU?U&}h)x|QUoG$pTqo1En zsJRG=%d&KV6`TrYy^Qy^*9t0a4T-3-ayO{2QIX%#LOLX>H%i02eq9cq(6>+@d364O z;IWvBDt~|Ke2yA`4HI-9IZpIdS36Lnh0(=5|2|-zm=Dirb)LdPwx-+<9niMw=4OEI{3XCW?e_L_;cdz^=tgnrEe4;<_x*x!2e{E14etRrumVZo?Fz$UR zt;bUfDp4d_q<0+a6ox93*cW}Eci_m=vEAW|nog|I@wR{Wx{U@=xzeeql-B;NO*DqB zpd;1MN=p&kOXD_SKPPCCIdPs{)k5VkThu<=wP4es=;1cFT(7Y>(FT2toSv}uG7Si_ zfVv&zm@6NZ0UofugH>0zhm3)${6@y~cb_9xiISuG*W#ROk}-Kl<#t-63rrzH|Mic& z6~3DUl*~&!Gi1>QWHv7#5!`aD_4@T|(EQ#^cQ+!wB$fwP#^vmasXXtw~HjXYrV=UeKcs@BbvWbF#~rS{kH zlc#0>FbsPd3#%+~&}H2C{@iQ1=XsAL)(#o4XtPjk}~h+N^ed^4-G35RI2Lzs6He_HuqBV2Rp+eBihd zMSUaWk?%bCkvXA;MBfAyAnwAjl|NB4KyFzLax*hCXY~zyj-;opZEYjY3_aHgnLGiH ziI0(Qw8*N9?$1Wal5hQSU&SId)NfMg(?n0cUMCp4iq2=ycV1vo?aUSwG%eUAX4>$ z_xWQ=!5TM&&(sXY+|<~VxQ*9#YW>;BDr4Dj2Zw|#jKiQFgm@g^_di3vB*W%^eQYff zf7`xXM(t$r1(T|69<-_zVT$l%Fv9@abVhoMr-;{-3@2z|>rs7JEV}Ga|CRL8-yVaY zS_A{hC+o$wxNe7wIX<3c4pc101|3J+($kq8K{BPOx+$RK%=>n6fWqpn0$N{z1e0St zf_Bz!t3>YP%R=bJd7#a(c*%pX_1tB52w4_e!-@rc>)4~GmlxAelRDKA6qxF`sP}pM z4YEwVt+|!sC8~Xc-C?_(I1@LaVB?1sygICn^3tylyx5EP@y_F$>pz%h-?m92PeaH* z-grG><&W2=op*w^^X`EC3hY!o~iHV6XFY`Ejts+?#pLRW$1bRMI92UVy<9j94muj9 zCJmK4J;QM=wDmc-*FhJilK!l&uY<#Hx2?e%p{Fwy%sbXTTax>eTEne{>RUf36j=dY z^y8wcpe`5hhH40;2>sbZvz<^4c^L4Rpy5`pp)whz1@H#Z$KskiP~`Mh7p(Kw5<*Zx ze1$E>N#$Oq?w@eV509yysjK&x9RRFjpd0#vL$i{sc8yut)U)FsJIVsS~85Q z*Z#z@`Ta=adD)68b&;v;(T`Y@mNsaCpc!a5kfS2|tmykSic^vvbep{Bu9E}Px@#L0 z?XbEPL-v$~+TZ`Y=w>QDv~YOzOeTuNSlIFN%b2hED2##d^qJ%WGS)Iwjaw>UIdOr|X|_1D%~Y+L9$ zMex=1fuRuhC#tFx8R=lY0w!GRr2Z+HY|V7!dm=2Da{h7G!%WnSRn95rUtlY>sIEPa z`p@z*9u4T8+w_r#7snF0rU_z(E-UG3lw( z#LDI&Be=u~zJB-U-vX~#nqzBgB*%8)dXQ(R39zOBk?;JDnW=FM*pT4f$x3U$sZ|!g zIb}?I1!$1}X%5X=#~+f00EI)qT*UM2{7Yn(i$}VhNmtlWPj5JSX)PUD zd-1>&YOX4Q=<-dl8 zp(heemjSjb?2qrKT?hP|#wY5YzWsG6&;$xpubqN?$Nissm4Hm@?Pg^3&n~zj-B$v` zN(R=0HuCrPQ+))Sj`eAwQHcu{Yziz4qUv=dY)zdcIC#v^S)nlkfL{S1hMSvcMbi=m zhhmZtnmBb~K{y%I6F%=N0U%tEBsuvDV}$aU^GmXn?2NKOGjQy)@P{T`v=M`G1jqWq z!~~mzemf7vwUrn|c~OWnBN^c{aY6>4dO`OnO?TGg;funD1*Anq#!njS#F$k-9I@YZ?jkbdC*bdHdfTLAEL z7n1{@cI9f8QWan!)q-O(OSzE4E&YC)fkFQb9a)`&wj_Emz+o4Fx) z&bq<$)2!u#nb7sOd3FeR+s0&zj<#;pyAN0w&k7Q#i(;RA*mZHU!<-AquYUwRf28|N ze78Yf7@3p`ySDap0^|#lbl6(dYCGurAFDUZ7i1WMK5ib@ZCc(7tl{M5(;o$0zvC#s zDll9iVsl2Fjd38G%9H9<2tFEo0qJTk#}+AA?Gxx?vrLK}JH2j6&3$B5eGDj^#jd=5 zmsw#Z_-bYx-vV^Gc3QWwoUWy zb)nL`N`y(ix#^r|h?m#(=pcA4MfWBztg*t82j+We63Zal!zh@|-|e1ncFusD{&f~U z)pw4D$I!&dQ%k|JDM6^_-y|$FmkhC&>3|g8QIcIV(r4UN^v1FXN5jSwORg+vZ9<_0 z)iiW@b-;%oqqp7ZD9on55J}<-TkG?1|_4SXf^OKC#_=Vld;!gOz&^Ye<=8UM>K%D8QdbK!DLo$->oX zmtX?h$oLg$O}rZ314Anh!WRrdb#VN8= zu(j$8WIThKZZm_UBNIL1ywrtd$avFWbuT?UsLeicO|8EQ?j-Sr85i}BuOpltj}{*x z23Y}2w7|FrfU+&;uesepGRM5#&>)Nt+T<&<`gJ}xC6!Y?+|Que&$Szo5rZb_(Y`Pn zu8VZNbTY_`%xM<{4AeL!v(t^%Wo30mShR2>fg;-(cF^$up=TtEr2t2(rJ9cctl+fYf?|5Q@!Ol?KyzgLv9Sr_OZ9^nTmH%|4NkLGuCE@7;qH z{;ljduOWptnX&*Xdn2!E<=AJ$=pf@5#u5CBXq2cj-g!_R>ND zRGI53_;$*z%&YJEIXZDkuPop!Bg$HzW|Psj9)L}u*NvJlE`r9?V%**W`~{I8pASD) z+qWZvCQh7Xa5V>%RDYs5%ch%Dg2{iSdLw84yt)AbpAv|LoNS%gc@x|AYnwc_FMXD` zbY$F-GZW`^@VWgs0nY+Jerki7poa2|(4hWSaYsUsf~}w3c<5D^G6mFdb&~jZe53W} z`lwA8bLoQC-OJzXk#NJtZl$M*h_K(bfa{NwQ&8quXH7Z3F%M0&_%{!o1DBnXfPoF& z4-ku~E~hj7A}O;@8pYslQhI?$rFaZs0nnRYm(uy>>LYGr(9~3@AwtPu3pf5>33Q zJpKwtm~9#YPqR@eBF2aR)V1TdD3vKF`}##gS(&c%Cs5n?92XIF=kuWqrG`{uKTI<6 zar*3FSyAm>_0`NHZ>_6+)&NNwJ(?Jepk@7KTA~v0BQ6Rgmt*A~a^H5-WDs5F3!jRxhsPq85m-Sh zWm6I`f@%i%552OzaG<17+hjHrY2cR)%htq9N`r~@%;|4-3d+%SEWGM6lNT%oJP$Dq zYeUxuGIb~d=BnV2}Krltn4sLGb5q8YV{5IQ5TZ1D^?gdv~x z;-eL?osBK;@)Q-^AF==DlCy}g=xBErmvJ$HWe}>$mCOx!l>O5s>b~5}`j$TXR1=nb1Rx+#45f#0WK-4#01~M4=(!qO_lfT@sy|eQD4XV#!eJ{37k_DaYHbAcN zU2#r{Zw%E9%RFfkBqni{b-C;j+onA}n@<0I$-4@{!@}v!bb+|78W2#|SXHn;4 z8ar+4OJzRN-RH1oBonRB*m5+HC-Xl);Pu&#eee%9b!`R=fZlhs1Wr}&VvfT07HXp} zy!->5%-5}BOW>hA7q!?u5fSz5697!SY(+_e^yCp%;BR4GjF|t@@l-rJty82BhpO%W z{+C!+g%whno}LaUhw97=QV4Rk#v#%a4FxYpz4+w7B*?0&1ig|jqYC$5$5@;SnG;Ra zVm0u#ZsWtZFl%VKud8aOaMMu%y6a8n!ofPY^k5Df_^Ns2n`4JcC&+WkL^CM=GtRDE z=BUcy_OL^~GYv4u-Wa4q^$M=N;_+D{s7xOu5hwY26mYQ$ripg-U@4fUa*D;~i*7<3 z6oynG8Rp3sDV}bL@z3FtZK6U?1r+rTs1ie-1;E$6A>sL?=YCs>3PGxJj6Jr(R(ag94f;G6IIbhaa5J`A|cH z-o9asCKts{m>@;c1>2l=b_15Ne17e;e(sz-sZ<**G?f^QDqvi87Br3NJ%!fP?}o)n z0*=Iu1OfMPVb(mn4~ZJ0Ck9d65{!7P_najU8vPQPD`sFbsvV);DBYJ2y4vkp_eB4hsH3GKjW6fah?W$zLDjFI z>`b}NYU}+LX>4+64H$mtIF#JqPUGf@F}GmYwS~ckQfi63?8U<%N`F()z6wfUyTEni zSPA@r9&(HoHIP*MZ$@524Oak^LG-{kuR@%a^vP{ZUvh0H*gP#77XRt^#tV}*M+lbC zndrrQG9bU8E%&7}HbF`p4$0IYoSl9(B07&8E~#qA>$lhG)H=!!-86ExF=S&+yTP#a zkbgNT8}rSTlqklQfL$s{=LoPQvNBy4@+goZL9o;PX`Nd7K881!b;nTH9Ah`kSf)&jH$Pkt0CXYjF5rID^wFI{i7dEstOt8r`@ zI6Ituv&7-x3E=Nck@m|-OY_G*!Q^=$v8qy;OCRpbs9%ov~4JS z*R;|Wf{j=Cq(+j${Csu0B4r&!ObG-m_0{K^+belFK5a$y?}}W_ ztg5bSABVYAQ0eYa5b2f%Q9_VZ8tLxtR_T`R?(XjH?(Xhx_J4xceLwf_INs0i{-Pf4 zwbxp6jydu?$H;G~GI6B5(fzNDmge$*-L>P|D5URkVsSqr``HMFiktG{=13=F* zUe+pb=HdmfT5QyOa{FmNkxg@}&&el@wcE-$nrNjF)|A%e4nrjqwD zQ1)GZZ%=6?+-f69`CCoM{y%PJXE?oqS{@lF)BJY_0clA|M1FG|MHlH9A*wXW9@a(e zL!uo0Bx)+gWq=g`wB6DoZwl^;#Q_DR8EZ5^5RGqgkl*qN2Vf?#5sc&dfw)wul3DDm zxUXVTSd*^QN>p zNKhW59TV?=tkXVu2ZC{*a}jszFjw|SD}Jt%vy4Bv!`PJON(AO0A*d4|Z7po}9af`p z*@M~AT9+Y@m%+kn@>_0!HyFD_aQf%ws<@Hm$s2|ygXsi8j>f~}e5>&g-+GFn4(!L@ z+7tXKJj{xh07hSi^g-GhlSexLIuj$*+`^Q0siR|gh{l{h4xM7qi&QdcmrNp11Gdd``Ecu2Bu0RZ=rBsxSvLiDoVT44pGb7q?jI=af!RqQ@ zEZED^rx^s?qrxrXizt?QxJ5|fh_)%@XJqKUBowng^B=D;#@EJ-$tz%`5W&R{$BmTr zcG9I-={x$9l|@TQ!AkjEN%G!o`hp2ACpj2Qt&mk0(|6{H;3EO?2m{46C@vXtd(12- z=}UcYe_oKdEz1I=%wM;}MUW!f{t=C{f>=tG5UdIrgU^9c=AYMY0e zr-8gyw8z&Gx5^O!n56+#lTT~^bBJ8iy~2;l%xdPU;$@f7jm9_sLIzTRjimQtn|5v`2j7y|?S4r@`n?rf{5 zR(f?oyL9h!0(90v)Y4mEr|vW06w#Uag*e?rK0yYpo!)9u2n#diQepC2EWqTnT$ODs zDLiSCw0w!UXbTGq9bH{kPqx9$Dk(AIcA6bmwHH!B7CE>ks&XrHLsHGc^iG%>=1#>m zV|Q?wgFF7>h^Hl=--TTR=K^FG6zugFc=X*R60FPmo>OOaB{=J=GuL{}y1U8FD}aj9 z>xN2E(f%_ei3)D~ToZzY9wVXW=I=*4Kz3#TK#?{uuxa=swjy`X@Ne4?hwE?uG4kfR zC-G1O>E^6HE(+{VCHeYb6CZ1klOMc7cY_oFTf(6ojysaI<0*(AsS$aH%FZRLsktg? zosycW5D)My={umrA}i7iI%x=2K;8 zT4$jNQOs3Rp})K<{fEo!=o>&yvU`fWso{l&A|Vcyp-9ALp2ngXx;Ib+AU|DNmcnD=O7%*xb|R zmqgU#171WfT{~fsk#sVf=aZ^lumG~b`ID`*XaEr7@V>-_Z=b*#feccS0YR)&8z=wf zH@Flnu;3TJU`*Uxt}BmuPm8GiN{@>Rn6Qqi^R85+Ov5kQstvQwS346Xd4EH}DOZSy zMH)SX<+hH~o-llP`RF8j(iwT3CJlqdoL)?hMnpGYUnJhfE;jD1p;J>u%9J&Ps52W^ zGo>%%PL6O&=foNnAFobIN|W;QH|IBOV)IXP@VL0Ut%b-B&@6U6^l}|x4kcKF0LS2_ z*`qg8KA%o6=D7Z&qq{yw`eW~r7?dkIbY>6%=49tf%$erP0FdN+H!Oi}o)9047ys=H zUi^b?fGmrthy{O}{r*Ej$9vtO>{I`3($iR5T9xVnu|luq@Wi3FIrerZW77hBTKbm9 zzSYgk9&7IZsQXya8I$;l4~#C>j}9eY_Vo5@2JB(4z=Q2IF*cqTTIpk!-@{q8PR9E_1I3~x&@>tCiY_Di+5~TG` z5CF)p%`&6fns`9XnPk2t)fWB(V!$~KpjlfB+nF*-tBSwZk*3FmJvI}H2p`R&v@`{F zWag}qi01)XM*f)F1lD)r<2jE*?smw>*KPJXHYQq5HI8a2#g=I)_OqcR0O!sMLQ)WM zj%AR$FC+MhCcmXD*r{>q=iNsJP*hCh1W~@SeQK8SKes}>(;jKd2`{QjmyOOpIlIQh-n;EHn-#zw zIpJ{eXq3W|zb3CL1!x%!9UD8Vno`rcWKAtE_0+DmIKWpb;_}rmVnM?gud#yxbQ>U} zb@sOfrGjt#bMguzzj(&-jf^u*^?-KI$m1i(#&V&Pmx83e`*B=-rVAhovk_lw0)R~2 z#M2qJe-Vww?hkjHJf-KE(v%n`_uCS#u(2-pyQ;1`Qi;ELOcv@W@>UNH27Vk}Ty*e8 zi^34~_xCRXnA*W_`*`out*tE*WvVxCn#4J2@Fwg*)`8&H>Kfp4dhepigLM3w4$9_m z_p-KJ9oT{#0i7p*W#?rbLwdpw01~>O(vjbF*i^9JT#zMa<)-WNZU!XkGPL)+YOyUMN=Xz^&Ev95KL9eduR2(g70H@zppw%(LADPu zP?UR%pwQ5jU(W}iLyc;&pOOqSuk0NhO4=#&=ImlYONWnO@NLU`AXb%&E_e@D-4ZUu zM$4%|A2n#W?249}eGz530RCvAgUDVPnlJ5b-sz7N11>NQ469NrC;z99Thc}i>ZWG` ztZc}4f`QhRs1Y-Y6?ixoSzRlOsLBmdFdQ|3%@&^g)DWOfjyd*1bqQ!rq1@VOoJ1*+ z)c$oD@_jkKKHAUyB8n+6uPYoK1LO5>ZB@qK@v0FyjMF zPIhjlBfXq@-Moqb@_#{~f6bNqj~_oGdW6FwB09K|)o5(@XPB?>WE(?6L+cD&FKu04 z#y(spLZTJ*^!2Y<1h=;9qDfqb zI4I{=$Beuwl#O@$s(zi|RVbFZkENXt9MvAt{)X&;p^(P9^}K}&r(N4v4{}Ep1K)Rz zYMx4+tTq%^r6=?kc(<#0+eMEM*Sk(vX-EEQrGY>_1A`hSZ>Fy1=H!~1n8(7S^~l`Z zT-%uF;*r*DYw^y6Ix+KK1n3i7Fhte?5L?GL?l~V^tPKfI#$>-Ggm>C<-a!rY@(@r?bj9s?l-%FH`jICsA)m}Quib-;DEvP>bUuUrdyOF{@Ro#6F1HW2gBc! zmkU2cJA+teBPB|5@HiW#e3kwqbC1}{8fk^zey)sSDHvA#B)VM`^3<3B!YU^%oy8rO zno2TD%E95_^=KCC0l?>RbI0|8skKr%Hd9NRVKBaI^4&E5!Ngssqo|>~g-+lnDKCh_Tbz7T|A%0p97-Qc{68lgJf`aX+So zZhN1VwcKm0(gG@TM9MNlndG%j!_sKr!_pN)hv*@+O_Oxi)ulqUDmdLuxmMSdwt<2R zXCJpxSYQ3-@bed=9+Y5x{)smool|5^?aA(n$=Qtng)etLGW~(5C>1DVUj^u}7wB_O zZbrf%3T$RZ#*-E}x+dC0$la!+gM$sBIm~>cYa-Uf69y6?COY2n1Izq8&+wI6+Hdmz zec?1dYa5%Ab=o}TYWPSRVbC#1-UryJFs}QvQ7}tMexK-JnJDS#b`jFB!N;W!a39;- z+26|ti5mVXv<1>(ayQ^l{(g6H2B}*Ai??k zp$k?-(h*%m(c*Oj7y~aqGBwpDVivT!MkQB(Y_1|lq+61&^Z0E5EdQ;re&~GYnD+qM z1G&9%1fIkRfQ{CnuH1PS!Tqm<1adfC(JOkd3>r-%nW`UmW&t5pcA9gn9Y0 zw7yM)0cG>tZ&maO)jf<}w?i>CI(k~YnV=cYbG7uGU}YnSB9Li1&^u>X;5&1dl`&vB zd>k2l4(B7TswoBNKDuJ0IuUt-a+*?b^0!{_@QHtNL-!9bigt7RLi~eDT0-t%sp0Pn zCyOnPE*o;GUAGO>!EDweD@Nl3wLz(Y#PrOdznLT&4^PjMay>1rcTRcv`PxLexw+Nj z0NkB)G`sk~VjFx1AO{*(uZRZ#8fFSa8h=-AHFF76KY&K?D!;e}M1_TVM~=h0iMIp9 z!Fo0}vQvDIUo0Qn+nal(jW}oICyWk&eVBVS?fb^g#Gu09bSBU7X29{XIAg+xtut+{ z^a9Y#?%d~ezGjNQxNB_Gd0(^H18d23Uqt^fWFjjcr|9XieCT9a%Aq7XiFdKe%IA89 zi*1#fr~b2+*#sgu^|e^y%{?pc6Lza2g8u7pcPQC%s+-&MyL%w^M>!z8EAlWD5rK1> zD>uZtyZ2MQ)0nX^h8-5wtsx1 z$t=LY`CC*G0Y#Nw`r&AV(k5^8FFXNa0|4W>QYVgioeTNGPt{GX<_vAvPO6G?&A0*%j9hydy8L8@mgPCsHaQ*)7K3DbL zH?rMZUr%p~?qP*aEwr}QQB{>l@hw0jqnTfvp1L(F=UwHZ#@n*RKCH2t*Fj9RS zxKii4*BNNKNJvO3OFEAaZY`cjmZ5XwNv22?C{u~*44!n<+ft$RXCBzZ9fw4BM^Tko zw4;+roJ0ht{aEeud*Pf|;sLk-_4lF7#Vui{x+ohu)0{@Ki2|2O#du#c1MDM_YU8mm z1tDI4pFW=)Xngq?7vSuHp}7yajW<+RobTT8%Lk+r-ZVmP8@sxMDq`|;a%QYnxHvd& zgeGBMy&AHpj{Wne32yEY+xlouj3>~IOiTtU9?v2uOzh5_9v_>>4>p312Qx2CJQ4K{ z-adkbYR1>jL?Nl!7YW5NEGdKhakkIQT}JZgzq_BY+_Warp>3?YT^3oWOI~Dn{KN3k zcfQN!JjqY!uXIF%EX4G)^PBVCqr*`N9&RvNFwf@tM0@HcywGX=>Uj02bF%Tl+89^W zbfu1Pq{DP(VWFX_%I>=T^}j%+vbR!ZQc|N#gIuk|`g$&SHQMRvi&kKs@Jv%b-q*NL zs^7rB2eluxQB4vwKH$)Mi0KbA2JN#IYLD~FSfT$G2&+OhpO75&oo)`!4qP~{xe(+x z1Kf-GI=%wM8TuNBt|sJLX1Kz!p?noqGl!XP-v) zDRH}4ltz!ZfZEQT7JudFqkso=%j-*Ijso(B&wmmdI-0W*#QFX=S&$1z^ z0JX06XE_zH!$6yYvz75xGVu(pUXX>5$rn&30TXWW=*5%`)Gmq-dEGE{LV)zzn5Ktd!6l# zgJyapzZ%!1YwsoS@ITyis`K1eb}in$d-$2q)eT)Aj&-s(Jul!8YcVVx%~Djvalav^ zGksYfe$aY;|Eusmegy9(+_WkAv}Gkk0#C3W!t=e+k@b3R`)&(eU4qeMw#Y`_T~AM8 zZ4cA5i0m%@P`ytNJM=Cq?cHiBgX@*`RqY~@>*0PDbDSeXAygI64ati-uEiViNX4Yk!E&8{_1Ctd&c9oU?aJplADjr)98nenw;9!_37*T z#8|dl>TPtcFE(TKw8&AyAJxQTqBwVKY3a>v^#NAnmB7_ye&vFsq8K%1sSHDIMrP(E zAU^2oZca?d8pr&HM$4iD`>^DOq&BMhu&!$0eO@;2Q<|EpDJB*WkqKqyBioOc~1S$;E?dS+F(yb6ah5MuD=D62>2#TF7F!c|Y z@*#NG462z~@9*j>$1iML3Y}dk^S6+7*wz%*3Lg}4JZ~iqHPu!0`dY9f$~z9O-DaFy zUG6f>;N2kiiBl6pL=S%a(x?}ql~wi%k74?9(7a);kb-W^QzL<#td*P<~ zUR8CGER^l`_SUd53Lp!floN{@tS4;Zb)T-x}@9SFIj;uVjoUD)ObrS&SIHw!_EXM*pmCC6ik(ziF4TD)EG;M4Jv z1cN`cycFb3aPe8@Dj`2ZUhVF-3erDfJ>NaDhloaJTO$h=*tFEV_qTR;GuLI!W z*8E383Ybi2w}u@7c;yyUw_c*ZHO{oOknwFYC@&m4y0@dG8Aew7~u|yX36X`w3%(-Ld*uU3G(w(<5k&%1}3Ht z`Q7^3L;y0jvZeykkw&3%1=_M=VH&c@cPC6^H348M;c-1%HER^eI}s0XJo-&tRV&L) z{8F1j0!+c{Ldw-}*vAsq6h8OE&!#S(2sM~@+`5W53d&~EG9pJHr zH*zy0X|km}IkW7{Kv$z&*S;++K$V^V{EwxZYtTqTeGz0Ki+VB#RhpvS4#|Le3+*y! zQKWM)0?xDbIlSkDWV^N*F=!LDD+4-};Dn?k&8+!TB(SlfcMIl}R#={Vy4+OKAN2{% zEY1e)krYDzZvG4^5|UWAP`}M4X5oO3o^Gi9J_@)pC=cl>pDI`x(lOq~1M>;d9mDd8iM?kYcabXu7>4$9SzHPtey zT7=w8N}3mw^EU(W}V}Y_ATmkI{OD}zNK@; zh@$ph{%kGl65@5v0*-8w2xI9VqP><^gmWiXt&Z&G>@8RbkzRBr!fbUCV8j;W^e{cH zYO{Ssl2$e%_n>l21dA>6w0!Q5$JLTX;->=L5B~v(|G-*-r@97_hs=lna>oI$1yHmN z4GrZc!K8W1tTqt5dzQz^A2docD;Of?GI231Wy5a8N#ZCyqIadW26O2^IWD&DG_PtM zR5w5e3Ool&j+csnWRB}dd5RnIh@sjbsi5=g#iy_i|DbCH|Dobyyl3{bDwumDhd24h z?WgCvii)lTv%{=|xE1fiUls+u1%O~xrNk!T2*w44CVRl-F|5@Znk~7w18Js(ZDjL} z;mzaj=!WF(2`^1h0cwkx3}Pd(6R^B|Er1)-cb37_MzBx z15B;)qjwKA$sUQ9|BTyPiZ(&_eW8681*(;T;@HbW^Hv3Ned^A3Y&SXdwlvs#zhq}C zNkpgpLpv+%3YdnQ7GA5s-;~<^euvJHB4sElbcWZ%-S|icJaiZ^{6CaB=fj*X&`o;E z;)8fdsEjwh1b*2(@R(pDQHsi|lOJw`fM)c8mYG=A)QskD?aoYHLFVPjy$`mMhy%QH zFE1~-$uV$H)_mXS8?xlRCClgp$`PQMVPi*9v_kO070xh7(j zZPitzq0F50GpWfxKK^D8%O7}+ z12}~|yAxVhp3hoG$b;R-_<1V>?N_==;9LzvurTSNO``RLr zgOBQ9`cO!nMFAGmAd@m$PQf9SXb&cPZTdXD1mj2C`(?VSVDDv9BTAb~2o3VS_Y2u5 z^IaCq3?(i)t88)lIXW&q;AOTwuO+?4kVn1hVO_F{*eS{@V`n9290kRf5sxdrS`wEF zBBjBtv0kj|*HDqLLibaeH}TL;apW5$p=Q^~Ofn?lQp7(X-2b zB;Rp9`_vt*i*W?~Q!oy|PSqj>(cC}@4`lzVZrjeVE&V)Ap#dy{1g)p8^XR413r&}y zmTOzJvX_YUT^V1@)H2c0rXYZHwiasr@!8^WP*a$OjhR+Xs%>4x zJqpSu;OSJAwE%=;%A7N9cnDLXsqtaMJuW@Nlmf=ZxM-XoARw*ptvHlh!v_&gzBSkR zI5UH$3z(~*6%lCO)pwbNiT2W+%#bhcls5&%uz2PBCSjq zu}b`#rSVAvBs^17RY}Q`j;B0JcfI6Vt!b#EJk}?Zu3kMSN0EAvnGrz!T_Cg>&(~q8CF*q5$qoQ%WxG?kq z>Hv&A!*P2-G=yR9srh=hFrIC*SCJXn*aRD`zm0u#1F@z5L#bi!0*!|cF|8ruL#3<# z%~Ji9+l~IlBvW1acHDP-gOxv`nezv0g7rTGHr!W z`seCB0;3+-ShhwU1mcYB0|=@37o)Cz*QbZ5L3mn@SNsnbnJw6FPSrnAQ)9j=X-~nP z9?JEw!)m-Sm5s8yqsQE^dwHb1Vdyv%&wdncT68oDB2=tTjVO>nSV*F)s@Xc`e*?jH z^T?YrFHIJpR`z0D4J{nv%6S+W(8(@Ox)^k0>nvzW4N}lcVaJ6Ft?^Jci&h*#>6X6n zZuTh=w+*lbhOg)93fdGHnEGSJk=vrNda5;(%rV-kYjV>-voHV1xNp*U{RR@#hnZe-qtI5m5=~}toEZj~Bawg0#@3eA} zUe{aU>#_+V6gjp6Q?$BtFz;v=uOxuK`Qdc;6)RI-zP+NKq=hA4N_2Nxz{!pY{0%

O4ITbiFZ!Yh$nf^c+9vdet;+a=lfRx~)oR zMOCcWl%DmC#n)!uhmH|nL^f#chBl)5j(IAZ%R}?Rx+1oN3~Db&xMNyzJgWP0qQ^ z#?2b=!;!Te&GGitmr@H8 z-tVlio%R82xFTrZGE$Cij@h6_Ycik}3vxst_5zi_w0#h=``tHe)}}?tyn1dA0A3}^ zOCoS6y7LC05TW&NAtEj1+ULyxj8_nOw-Hz3x-v_t5{dZ$ZGkSS%iTFck;v3R98AUGlsil`Tx; zIl&wym9gy28&x(h4+TW=JbG3(VeE>Ovb+Gi4dVTqlrtYjdpOosbrZE0mU5j-!`Q38 z4Ub_Q%_4tS3D|mUCG1LWqMo=Z!;zJ+Ili5YXP=!H5!$alPMR( z*`tdM;;{X+NW~|a_bKB`vs}TaG>?(dMjKhu@j#RX1+0V|Yc*kQdJzUXr3qhy)amys z7WUwjTqATNaAN?Yq0DCt+VlYC5Hv<&Z_2k!Z zvjeVN$J}d>=Zi6{IG-$y0hS8n*FYYIUNgcLy~+ReDH|PW(jPX~bJ^n<4{^W|zQvmR zJcc-0^P|ypYhh~a)!8#0&B!T&e@0E2U8=Km-Y6P5w6E%5v3vR%uaCww*<};mOj*4l*nX zG7@wg`rpqy{qJ(zyx`s2qVew8sad3q?|c#74}b=(@>@U&dz?RphGH>3mfsB)csJ@@ zuV+)x8B#{7?p;-&vgJ1JhXiO3;8Z%|g!;|)L&OPN#ngFREZ+_;KpsS~$w~V6V(5>| zUHG4YE}-|5yCWemqBmIQwYFGX9>SuVfsf{Jkq<)N|N#fcZZNNDG8=#1==6_98MyyPl&Big*xAl>cNBo=8C*DV1QJM?s{y~VnKm3g*sR?s*4!Dc5 zl`H)@J}qCWViKZ-sV`d%s#ZL-jdjG{nX1G04nO+r7V(*no?+EWd=%j*S6$uWo%sLy zg^$g6y-#uT>r#I*&rV5YC&vf|vJ=mYYa{xH%oA;D7TITxs9KYs^}`OEafd zmd!Y4$2dp2@gzV;7~=w7LstTnO@!1z01b2I3l)ne0a*hwEulZ+3nlfNL@_^$-^b0} z2luHkRR)T7eMRl&x+A9VI|e-(ff*$9QUcCr+NKPQJ86uNiP$n&9RqVPv2#88Oz+X$AS^HkVQL50&UZR0%KJ zJ;=0iqe0x7(9nVyUDjdSu;YyO^XSG}C^)K>%agAw9%XJgI6j4P?Y z?`HV~ftNRscOQ19IThd~+x?jYj-AMiecR}Vl8D_*r zv4z8DQ;I3JtsIFub^#o#Ey#<*w{@b@J@Ji#X9NdrsHG%)Qg@6A@j0KQ9k`F4F%CdM zb+I*yvUyAw&HJ{m>nwO(6Ak|wfeV@b?J{T7i!u1XU4C(E>Y|Sbd2sCJOMDpo75;+E z{>N?NN`%3YPc0;dX&_ktb6%a$AG-7KYY4q4nJCRSDYIQ1;?=vEHLTa`46FFa3B^wB zba!Rj7s?KEO)9iLeU%31xUHdBsvlQH&98U%J>anVa|iCOPt(7kIYN`&S8EHFhqIc_ zwg*HNh;&^4Qbmsp4KLAMd>g;WD$~z4%kChf=(?G34;f&%$7LFDMLI$Fy4=9LxU9KZ zmAqT78>bK$9vhotaK}9{MHeEHuM*3ckzoJ{?3=$BEF)_WUw~xDW=dSKT5?FoQt4Lj zHI}ye=|DpJ2&Tr@ud)q-G#IY+#&hEDvImXar5%k*Qn^^>Qn! z9CORQr_^4*WA@mow?KjBW`~?M(V!{`5dFO*$uQkhngGd8>V?TJw1_jDO?cXCc|@ z9)@Kk25fI{0d!n`#pf!NAsM-F`r{8(gm%ADW}wObn1AW0o~hd3gIwxO>$V2ia`m!7 z;t5WD);ym>m;z3S6zA?W{rgS3-mTtU8W=60xSC8eKsFPv1*a=44+$olax!noDQ`i) zG8aOphr4Un{c$AkuB9(uCP`QOB=>K;#HoMOxzCOd@<8%juhvqx;uaxulKUx6lP1;o zEqD_3YRfIoF9Tr&w53Sug8fey5m4j#jB|5Mt2S8UwR`3EhB@_g_P<5P*LCigPE}oB zi!I|FU&6OT#0Gx2Tnyhk-x7Lc2#wcOYc06uAZ#Bm#i6)*eDdOPg=C(_aVVDm;qjhw zxGQqiwY^SRBQpEo-gtZUh&H6*MO*-J}bqmoKe9zmeCfeBpq~A3_Cn--HA8fc)ziNCF=;Ww$m6esX9~T5EesN!P{(OffqXtzab}OO>nlNnnK6X)rwOk zClt_je9>VKO;$3klm1?~!G7>IC!Wdq#y5}Z!$;#)0uUkYf~+6gMv?%T{alO**>S|gBEO{ZI*Bka!n z(=y@|?@Sn@=%`eF>WEFv+<4?#lIs$jFiDWLu{i#~`A?-$>mt=zay3HycrQLF8{_C? z`(g65Od($2=jPYeJY|(`u|cU`jh(AGJSfZ+>inLCxjgZuo9DH2N5)_aprC>N*d_ zBii%y)m+59m2g(*Z`;ed>0hPBz?!$-|Ls53ayI_)IAeoUXpnI%vEY44KhRk*x8+aoo#PoIP(koilL9sB7eQ z81^r94CAKiqZ&xDY`E_7W)2>^pX@n&aY-iJ&CVdMJ|W{~;vqfSr5qVcLG_`1a0XH~ z%*TDZB!V)SvD!r;9BF);D%XgBqwDU}UM(UAnX8Lm$r}#ONi>;19^@y|^`{<|eWeu_ zYz_nU(=*n519LIet9!=KLL&2QNv_eKiFsNU4oj_WHC zojAxx^}<%Lg6nnEb@EDt3Ak!+M{&SD01+CdCn0AvT+J|^Yiq!Jn5q*zT&jq7yk-(p zt#d*^@c7h$gQE)tr8k75n@Nv%U@mU3Z0ArWPA8-TcVk{kV>uWKjN++?<&{KTB-hz? zpTbwOkj!#OHk^s0yjWt@Z6W(A{`FpvrQeHdN!4g*JY|9ZGvn~!RFvkesR>^G9k!A^gzS;DJ3cdXV`r|hF+u^nFaK$UqeedyA zm5ose_yD6;ar!6oqYP8US=cV#ru4U%-W{`Em=oQ^v`)=z=LkQ6 zJO9)8P90{_;OD4>bp!rPtRuqmDmFArM&$@C0Y6y^8RL4|Mwc9xpkXbi1Il>+p3a$7 z6Yul@NHnXp6*S_RfmR#}0?XW;<^ZnhwCh$Ia9bQ;Jjn)Q6_hNUx}h;p!0Gr03Q9$_ z<9ec0+k^6<)-xge@c1qS|6#a&l(R4ETt24v*BfdTR&8(D?Ao^RopAVu}@UaDeq#^Z~>S@r+3 zm;`zKxe&;~)0i9>C@4Ep9Nh;!?GS~dv+pCVwQnOTiBvZ-g0$*14ulb^HP%xsl3nZk zJc5#=C*Mwb<7wALlPh#gKPX7nic0U`yYx?KLZC0}C!Y^m=Axi-c-?5n&8O}usH>P` z#-H{Jexas#6K-B{!ueTHm-Ih6h32c)T7jZ`%q8{qp68!5?SRKpFP^2$2pgee}XN+*@U#6H^E6* zuDaKu^k9UqMv{#Fu4*3eNVlJG$$U^Q?fUQ8?9}$e;T~IM9G+to{J+Clw^a}u6L~aP zi53BTGSOvB`_ihspONMtA3|_+EAXXwW%=3Z;OHw>`>3>D-JiwQJBvgUKB z4^BwL`(}fZX=aF}|BskwA&0{qduQpnj_>+4?m$ERc^(u(xRQ`Av`lp9tD>CgZ^`g< z=kI(S3Qryn`HNzHe?fkLFM^D2O#!99I65akSBg%7T0%UOyPHzP@1BMV-c}k9_4SiG zPEK33-z=3v+zt%v);PEzF-ysASQ)F5rv24TV>aV=3I}WDu!wMyY%o*_S%&1@M~Og} z=qH0RsJ^2A^QC}*uVVB2$4f1iRkAQl6n#+;CHjUFcZH}U-W2}5h@%?$aXr4_%Qlc5 zC1nXMqw55yAV9srk~~E7h2$S#Qae`_$rrMC-P{$yMA=b;hGzAw<&l#gCoBIM77#s{YeMa3{Y?w=z6K)_BPp`SouAl8nD$c!xOd(9@dqhd$ZNF6 zeas3iyGsM6^N5S0QGLfAYOo2cdzz9`f=m={Bo6M$7}=ZVan6ZhlAA!suzFzPUk2MC z5u3cxA_}${s~hVY8yj01+Z($oj!w3E+tW^b_B9&qRWBcXCCsy5c77z3)t{(?-#l&R zo_Pt@3DCnwl z{<1^IGv)Fb@MpPt=ukBi9S{vI@Fl z;F>PZWFh=mNp~)A&9}SsXQjexRtGvyf(26oD+LS-OvSBw%8~p=%NRc)V~)8{_o}T4^e8!p&6&d$r)a#p}v?BAtC~ zi**z4^?5Q(L~l9dv7)iMOWyprmON#W0!RMEiy&JBkz>~wJg?9F;OCQ?-l2XK|E$nA z%KI)891)GS(5X82vXmCoVB-6NC$}7Y3#_PU^tU+WZxLbN@xzt86%tThxjUstwW6MA zO2Q`UM66-F?(HS@y`%zq^6C3fP=6dXd>&_x;@Szl3#nWNpz%jt|L^Aw-jlcT@4tQ; zJ#FNF{<8#F#sB;rEF8@LpD!d21UxfSQ&Z5Q@v)?S%)T82!(5g>PZoR)eQy=bVYPL1d)OEm7G{=M zQ(b+&J6WjR8v(cyV?}_PV=|G46UhcB&ij$-J;5 zr|kiLJs6Ul$YE`8y3r5FTI>7!%8{XwLhtxI;gdsI85oigwvDZ<&irwh^b*hN<>RBG zq9FH&u2!#zh!O|lIVECQXv#*w183nnJFImDC-ZrY7)DsBR2bn@(hyk>7ZnxNu;wKs z5XO&yOr0vqlehOIE}u}s!os$;xAnot(C~1!a&UGQaL(A?yisd$e=Qt-bzOzTbN5Y5 zEI%pfCl$Nd64dzhaVL@LSrwQ(zSjj53Fz6*U+r$sdYetdFUi!<}zKVOLV!}9!ln;pr8QdoS2xH zz0t8T>72V>x<=4vbV$*_#sLotOR(^8u~DKB4GnFYP_{q`%(`ByVGRZ?hf7Kr@iOa7 z<~d}m3?~b~`dM1Tz+=-}Zirsg09gk6p;ZV5QY>al5OGNC{0SoB$rw#K_`L4n;c-S_ zY;2q!*xpm9+Ng3%VGeOn*V7ZCd4uXhiQA<1*Ny(Wum{@O+QrGKOu7Tq)YNOP?=byd z!ebu+&bS%r^z<}X9Wds4Qx3?$$ixH-3yVwgYD9QA0v^jA7>p*v(=#|ICo7vF1sF9e zu#FElhLrlV6~?X6&wo{0ti#4f0jZWNvEH4~AI%sqD_Q{W00u6ALEP0Q^EE(-o0^(_ z*feN$haAk;mL9Jf&y<0)*45XSW`oCNXExpgm=<7jtZi*|O}Af&$FVhs3`Ei@ujE*7 z4XF>O2!N3yXAbu_=SH{)yw+E#lQc9m+mu5XnUk?B#yiucWR)*#=D^L=3(wEaHpbKd za&HdWcsQjUJo>b?u7|42^*U&TUo9@=#lp&(IeC11e0I|p&9s+Vtl24^_;J3Y!&61I z&MsIyfioZ_B?TOao2x6K^5@}~+oN92%?9C-k&R$X#n>_!2a%h*>y1J}Nm;rxR=c%Z z*WBR#e~P!OGB1xBYzQf7`l8Ei{tPT8CZ<$|=<>o3MMbG~pqCo%lI8N`86HJ$wB+(& zUekUgEDZf2x`>xHb>{#$Gc!PgmG8u116e9-{joul{BbUs@1;SUySsb2;UotKM}@)o&gmeJ>($XR-k!++ zs8%PdvvdO2-rc79iRv5*39*h%80DO~N7=)P$ zV?|zGUa@g-Gg)hjMzCEF?H<85#8v>D?(AF(UZzGa8I#f#$*`E#TI}A4^M0stkVp`V~qhkx^)N zbEaQWrY5kSwlJqOSmp&*r`UYR2`E&B3L}N_RWQiIuH6@-JQ0t;3T%Iwm*wU_aXR`U z_5bL`z+xYVYZ8e>5kEAsS7n*dIERKa+v=$EQcY*BQ4?gSkmEkEED!?&X_0Q}kg}zb zZUkXyaOlQDB$XHt7(kHj?hr(f8gl56?i@P5d#?L=uKW7F@B1IT{>Dd`InN{ZUTf`j z>_bjgmXeaPuC8t_b8c8e*ke(PXMXEC35o66pIV3jZBhIdF!QiVzaQadjkBpdt{vxY zA}1S=cAARkomZ=NU~0fbOD*R57+V_2+yFN*!VMq!eS+4*it6eqPaW;-GE&j!Tvvw6 z6kW@!LV{sU1Dz+Mm60p+hzGo8r-2UX)LnIKcz>$p>SZ<{88m@A4Qc+Ri z(5HvScyO?-hx{grt{1j2w1+c3p!-`Q{11-#o%i7VISBAbvZ`6bZkE?FQ_z1AB4Gy1 zwF(|o6+8r!KLkw0ixi^wUo`{>+s)Xoj>gm&n471Um+uRXxhpTf=>MUe?zK89rlzE! zu?yd?CC+t$P{SbXf7Qv78DscGCv zJR4kQ1Np|5epcqo7q2sR#10UwL1OMjURkpJ;O(uda=A9S`;~CFUvtrch~rYJ)sO&d zjq4UGg#`8W>jkErci{#b>ni{(-Qg;ORB{Tk5db|ezCYEib}tKJTKh$+Z#j_fU5x(` zZV_TbL)wCUwzE9cINEjb5`{njX1Qu-^!r2WQf^&F>rY1z1<3i$H(~BKX0>RYR!5U~ zXJ=3|!)10p&1El!+=1)C9_VRnZEH)Dkd5SF5EP7Xy@nmD@$xDfd)GW5@gY;cCRQM2 zR3>NM-F$!Y&Hleo=aj0XB!NtUG%-W=S#3>ubF(6BhB~g#B_owGXApo1Of{EYr4TP~ z0cdN*0ECo_AXEF9gg&stotA;a$p%aUCK&W|ON7GLd+{(cG@iThK|z;?D3}BVo%(YP zM#>#z)5J2o~ZTlCyy1JqTZKOG@ zrDDZBq}z_M>iSYBCI*HVdtbRDCB zUR7yX2m~zCN3hSg!`pHn5O1(%lUiWAQxs}C)L^tAt%`e~(y|W={C8u&tK`>(m(S z;qJar3P)aJAc;^NU#ip8)CAN86ZJ=Z2#%>PJ3^W{d?=82F8ac5pS~B0+^H43(5ux4 zoPwrNNt*LlRU=wn5lsO93DgI?{oEVK+#mUTllIudi{GKYK-#;gDDvn#@3;j75Csgc zd09JLl$xL!d&Afs5WIO=Q8BSH5`nlg9#q@ma+CKURi%$v7*Poc3H5aC!ii?!cfOxK zr5-RO%ZcyLegE_4&$-c?0H~4`;xKyM-qFH#$vv`dMc?JZP$;<4Bx&YNvOgxf=sA75 zv0eEmK-oDhmHmT*gB}Olx`n2l85aN77~mm@F?vymK;Z7(cdQCAA}D=m_8{e*p`J5C z3Rj_p5bj;)j6k8%U=-8R^5PreOm-&AB<1bfzcGGUA{s9uXTFzs9c_`;kT2&dr^u^d z5Rrf2*%im@eXkE_3phtc-_yDq421mY0_;nuT^!&uD9Flj-Tdm0W{0pGP}HOvzsXbTB)61x^7$U^;rJ z?0B!I2+k;-`2ENgRxskh+Pb@oE~f*r0-~3b&GH2=@BQbWi^J}kLHH0cF>#iRhX7F{ zK+Iil|63d(QSk|>si_b!VRKKaR{$SWr4V!FbIZud(eKVAhi+J#F94t=NtZh;f(yR1 z?)(*M#c{XQDyAagXxg_F>6Qit4{V4pUFsi#!7jKyT5Ls8a&fsB(u4|Ns7KkaAuN%T zlMnY$8tre+A5PgmnJ%mYs6C!TMM^@lv4Frj(^@9xS}OMwhfCi1hOoh=qf~^W)wBUX zF+MZs_Od(qy6*Qc4Pavd!X*2oJ3MgI0;C8eFG+3~o0mBP1OZZLW^`<5U1B|wVD5Fe ze!gZ9f*`q|H6=AQW>ia46I*Ee2iieBp~=$7d&edvCG`T9%tMeWvcDvqB;N9F!C-)k zYC?zmBu4xR=ZsVxee1>`DqckEw-9*q*V=h6uz<*>rD4m@eR|axG`(60KPyev0xDoB zdZ~*<=C!I{&OGn?INzv=1Xxv?ZL#I|i6&qb2q0QDD$r3y1cmPc!z0{WXx43N15h+) zv*O<`m|j_l&WdM3l}y7@dW{?E@+vAU1coP>19$VRj84MN00R1~f`^Z9su(_O`kZtD zZ+w)>!`as9F=)Xs5aPS!h77%re=(SCV0AWY&0P#%$o(+cgM&pDH}jbwY<}gee|MHB zv6%)?)-+F~V`TPyheWk3t*Qk!|9|n7JlN$8ffNO*$GaIBH*elNf*9;i$~7~?kH9^F zoa-&FRpH1X!>M268AAx$<`5JJU<&zBuq3`e3~yzCzJzVjf@2j8|7v{Dhkd8VBS-g$Z4>PkQzGBbxE22@s7?%`&A zpgn=Z*_z!6qCp*X^$68h-_v0pN{I)0C^s z2@w$yC)iHLS5{VR7NS=A^WKxYPj~?o%Y@xllq1VrqY>@q;v$8DgmCD6t0D8OuHqmz zQ?o#C%jw^kK^tx%)Sk-$Kqqm$JGp?(NRf?9$jY)ZH*d}fN26sO0NM&bM6pI(goVs% z*`3_RRdZNJ(hS-Is8lM@SYc{*wgd#9YjjUlDx3jBQb|x~Xla!LErx-~3*e>lmRNrR zksnl^v`U3IaUh799Y212gQPZ8VzaZh_U%j{Ilo_Y(R+ZJW`l(+R@gE-6G1^iM^{*A zkjCI1jq5GB@b%sDpb`8S`_e`^0(b*-1Ee=(pU$41M4LW{Iq-qg)6+-5hb%egVSsJ1 z8#vtjxBHGu&UETHxUfT-e-{oICvF@N0#+r%Vq%VdUXi#&Evmuj(_ikuRy2-cQ~$rm z=f(p?#V>FpBP>r4s;2(@`9W+ogWK9d?LWU8Tv}SXW^!JDm-khu5RCFJtg1uJ&q*D{ zusQbO?#h1cL8}cfV+3Mjv3n>{8b(aVKw8=lE{y}Up2%wV@X;fik#Zo~jVNG;mZPC~LBJ8;IH9TC++InTNCaC9f z6aSy5F>GIRbMFBx+U6ouiimAC=I=*qecr!#vPz)$FR_W>xajrx!!#+hMQ~)~01TS> zGs%_;m@7!w)JNm|w*`j)dk1Y*RaUz0 ztWc4Yr@vhO5*>ZtefKpqNU--+H-~)!5J9yCuRy_q{5>i7ca8c#AM%F=z!~6|L+s(g z-M^>X3@%6{?8R?ti{4CbLBX8;7m|`c!25uS6_d%~(k(YaAU8KB*7ujFAI--JTW&P$e#vIk%V}f9izxBD=)A18{r!mn56{(qQA!LB#=T7 zz@BUUr)xk&ADpE5z}Slk_pW(@*Dv#_}61`h?2&mgZoXgz?2xe9SG zf$;2NVpXv7;ij}+y}g?r4}Q_%K9<|f${QHal5b~`+`O5To?dRz$H{m0`9E7#a^<3T zooa?!T1E!oO(Vz}kPxBWpao=;6uIaItdyJ_Y{F%5pRxa(EH5qXxOU%Rq44#1-SSY$ z6&mqMfG8m9Vbvh-MDrL+tEu%8QQei07;9_Gf)oQYa(sMT)T2{qdW}jq`{c&a7BefW z>&ej$Uiczg;bmc&uUM`2_w##*s}=*8pPZb0i;5~RnSkY=ar<2boB_5ltaBs-KKiMu zjzHfy^@+*JM-JtV%dDYvj7&`JU%9_Sl7JSC!=(tv+>?zBp^40FV|?l zsr$PKj9+Pz)v&KDfQDvd$V}Tk`%LXIi=kS{dVCiY&5Oju9(!~Rjg3w#!5C0J?NI6qkQ;t*>4W%D)p(=t zF3@lI&@ghPw!Q)rg?&&A-&Aq1u`P~N2*$`(eRjs{P~*@(O3(gLo`xu3RX_ysp_Kxl zvn>NMB3w4-Ix~cZK|q#l+y`Z3z7(r= zTj=#2)jx~@)$V&dRUVN|x02;!q}`s&%DTsAK|fZgc|h09r%w~$tMC+6AfidpmEeo= zza|oxQz~w30($nbXmI=eOu}hc;mlf4_x5-#5(y zgu_CQZxsVj^g<~>QI=L#WxAcmC|2Fl*YU}r^Zj{rs$cIJoOU~alnno`WuEU3G40W# zpAZc$-@f+;R)WgJ^0Jui)Ni2vpa6c%W9ItwvKw2095tK&U{r9po zWK>6f&`xl(?FDt9_I5v*=TlKt#RCb*;SwdpX{g>TZ9x!Da;o>!6Ktn;Ax#`?Ek?w} zwPOM4#OnsqNa#I#=5zUh6&AESwRk8))YTaRy#*6eByU<`V&W7utaF2%(ilhqJ-6y5 zZWWzVJzl9;RRh)q@&N40vw!_1>ankqjv^!^45Ai;y`2r~tke3Hq2WS54d_cJ`-@y6 zfebR?vN0kK0BMm(W@Ca#B<0*2aHs&i@XkDXK;>kZeP#yp-4f`s+&w2o%H41i1(i0hb47sQ7G4l6sE<_7M>lu7b*-%G{qnl2yjdVJ!B-T7{+$ z7@!4^(le<3aBU?-ef$whM9SHk9t}Z$GKE5Nz*FS*fzum79RV?P48QhNZk=33hgNjuQ^3 zbi%tVS?TFf$7sM}+NFR3+vf$HGTWpTtKagY_4QLJah|Nq%%47gmhd_jZQ(lD+bc5b z{^)hN&?KFm-_;e~+>-i=`H#&_d)7!{J4V!6#Qb+I83uM#s|OV@Elu^3<7gscC7*SE(QjWpHAjGYup>@h>ba zgbfAl7xS}H2Qsp+3n5x>4}$hmncNLgTQ{RTeldu~ilFRY>3nT{-QbbOefzmPMG1Yd z^3%|9z8I(iy?8+e8vt|#m|~R|!mGSk%_4IwFhb33J1D%U*v@w`6a#y{*uvPyv#|+n zuHp#`O$3H_d(xCQ!wEBbjTTQ)3)_AO{KyD|A~Wje5nL0Owgv+PMPwb?=qL5HwQsAc z9GA(fjL+B#Lzq`pQ$ye$_5_3cX0-u9z|GltttE}$6_yfUV^5{8n9$2`TW>3>gRB^0 z3~-h5@^ZX%>h3;H|S-cPaf+=0fAkXgg~IoR3ZLi8P| z-(}1=TJX4Tn)VhiSv`M#VW$%gG0f8)FR@*`cv0oy!K^fNO-uXGMqHf6{XIyhC=f9iIXD7Q?ps)Om5`{Y=F$AW`0%i8dk*TjtChIR zs$T`H$k_%!0jL_uHF$(=21Nw6qHd)#cLZf$XJ>r#VdgVW&uUPFZbB0}T*-1@4vkey z+k@J!VSHua>|4nWsAA%8<@471>^KD27httCYIU?KdfN+ft>-vYpCKalx8A%&;VV0x zq7a8*=&@I+qcM~PsGF6YjgcF$G&5u2|Mr#GAF(J2Lk<^aw}WgaG2UP|0wV5q2V~te zN6_+(D}aX`x+Fb%_%L+?Xjdsoc*ktXEWo}2(lV!2YBqz(F)v6O>Ca4v1Y$)T7O-15 zfJH2)-pF>^oYOq|fOon5{qIf=u$T-+FZCQffFV8A9&Xw70|SxLh}N&QIZ(V7Vb!U2 z7gn$Pm~6{y1MUZ6;y{=Y+nfD=0Y7Wzm+NfV2vMnRZ1h1<-gnF#h;L9(_Or2|8Y`mG=nDs8nLCr9qTeuQfW{z(rJqG zI+kLXZI7E(U&Lla`1uh=PTs6Mdjyjz+X1z6a(f@M_5?2aOsr3rj03!j;3(K*Sj4)ULvc@ z`MD84Tu9+itHy&C_b68>tEjk7vERmJ^z#cq&L%FSmUED#Y@vrPCd<*P3e*u#7;XsG zk$h$(F|1x>PnnpQ28}wtJvfv_icL*TRYJQsrAGxoK_4?%_F7g$Z&tiQ)5H$p80vM0 zoozQ!GE^|+TB@q3IDb&abCZ)1MNCb}uj4&6y31Z%>CBFXbTr;b`LA4ZCkG0_kX4_i z)qVPOA&vG25P_tS#6)_jBtH{Ptz!-KaP=c-s16b;?YZc{h@|+V!^3_WeLK6WIJ~wg zz7JHp>{|8B_69nTDf^UA0~mbK5Cv(_c9A%L&!3I;JXc0)>e}^uOQgi{pJ_;HicOF( z=N(*Jgm+K?D|D26*)pkQ?m`69O&RXcB>Pp}qg+}BQd z4FMAecLwStkd?J@)^V|S8L%iewgJ!hW)wYX!J#wbbY&0=2(T;!AF;I?zKpTMBD~Hh zo0=96eOOI1)+Zb7@CE>TT}F8{^A`3iNEeD8#jg#EF4KsMmRsx1_fQh7{Z|#dlLL>u z1Bgz#Z4DOk3+S8!z5BM+?py_Z3*VSU<#LvBC{KaNq5Wt$hjwcHuV3D5dlUX#ra$2> z07&5#6%_>!1q$r2!6_MPSqU3wiEgbG{A>mFaInyfPF2W$P7x4?b={m_b8t(c4@?F$ zM=Rzr0KpZ7ucu{%0#}Dk4H|u{MC~u|-=sysH@|#&hoVuAI3Yb<#Jq>u_whsh$gII< zIS?qstOoh48?K1wX~_Pa3)8QEq!fDnX1m66fM;8PhX;Td3RFkM7cW>?Si{yoQ#<0_{V?*tmDxs zP3KNIB>p6s2+zayMyOXoY}aX>3RMsf7Eu-99>gWfMfVRu5D~B%3~R9{Pyj9nSIu#S8#Q(vX@C$E=`4nz zw$wS0&+_amU`u>ZllbWXf*XK85JKyWtL8KC;JTWQKf>Gl7nG>ll-iXz;=;ldq}{o4g&z7XK#4|P zULL>|3Ma+(54z1 zCd?KqqoAO`NGBD8M$c_XwG9SGM5H7oP0!BmJggug^oFg8iV7tshn*_`)jwN_n$7Ew zh#SiF=fDKDKI@p{D|yGcGl-b*%s)svH5B3wWcD?9A2=662Gf8K3xH<}<&pr*!UY78 z%@zai{r!o&NXARre)*FEIxu_r^6HHnDvFA)AqIfxkfl?$A!iJxu1?@4uw6_I4FPRO zef{bT^)rA=QgV5^q#*7oEfPQcX%CAP3iO|{BXV-Mj_!4R=4iREe6fans|g0kui49F zg6p!BL>=HJu;txE@uFmw(b4T*)gXCX3fKnaev7 zw;-XHHrr7CgWCvlnt*Xag|rATFer|Tz^#B!*&K#J1sBSH_(_J%4W(AXzfS8gK4G{) z=(e?h3JIy>Li3r0goJc~KdpZVglYJDGjsFN3a9qY&O88Z@NB0}omx^}D5rZCWbfb* z1Gh`VTmhfNFlq|$^MjCv1FtT)X~^^e$SvDJfex@@&SV)GIXS0(^&yBn_8+0-18)d; zA76j&#fCkn2T|)QuSrl~AaM+?8O1h#2^On^y*<>qV?bsCO*w#C3`&9nt*w{(_S^m? zfJ?4JTk;c7nUuU?!GJFTJO~rgt*xzEMds5pGtj7a8jMZ*Jpl1N_?*}NK-QqX?+6W5 z>l+%no~fiH*fQG#iBbT=U^-BaXOA9z2d29++Y6Q-@V$2XgE*7x8x2)6vlE?sZ4kgP z(#$uy7C|HH3fR>PIb{Ex3j#kbF!Pz;65`~Pmy()? z5I-dy&aTO0)^yk4(9a%LOlZ8!N4NS z)#l0w{^8A`Su8HJXD#_31q9AZgaadjkpnDnO&wNB1c=@`^dzb(oV>eWo66sSI=W)p zmoHbhdoh@x(S~2YIu{|~Y}oalId|y~;wOHQ97cgYTwo5WJdh+vNYjoy$ccIY1$H(d zrzeBYkL&R02mtER^z^T0Cb_6hu#G|`LQN$IGsBcNF87ZzPeAa8!vRnsloBEJI6IfZ za?Hx&aJvt$2Ns2G*QO8#kb!0d4go=yu45qcXnsKOdGPD;pag>gr;~H>O+I#bUz3Kv=5JZ-RQ`rU(Or_i|_^ELiYXDNRP; zgWRREqN`xe0T1TV@^YX7#^zu4b^;r`1@{t_NSH(!4m@E6P`ln9Y-^K~l4=5%EKpOq zVE9A!;OFcd9N_I}yR||GCF0FaxC%a1yb4c2ii(QB4Gf*Wlp^fD5#!~}8-!q}n={fF z$H%!sQzWQ*Kuy&)ATMX6r8&%ZO#^0uO`Ev}$nCa(%fNf)@^Zevb)esxiz};%fEm5 zR#RIWm6)sj^XH#*(94jk%AHm@Z}|^sQ&2<$?a|wT}UZ|+>c^*0!YtY1@ zP&c(N=AIIPrl_D1J51DH1dIxr%0-Bq09cZ_J_DC|`?eN1 zu+R3X$bZT$P;goIeVo9eoXm{{oVTI(=QvTE9~r7 zAB##k)zG+gv(|4Iep|m7XXhiPKp`rc*M3wQJH)M9pFwCodi`KTfo4s}tG;C_EC+?` zu!>ex=-d7JDYJ`YT)XS&!}LV|rs2-o(xP}8W43P!>ojJ8}v z79W-p@}<(O(!6){KZ}@>dMr6El^tBXbtRPQUDuh(jeNr9(OB)*uCYHP7AtnA%C zg;&BI=5BufA;aWxQg*V^c(hYA>_tqi^j^6@d2&df*k$ep#m%S}I-9qSPX7ffsDVuK zfilP5o@OlddW4)|d)yPg!cT(S7<(tSmNwO%6(*AK^R7%3R6Ctx3(!X0Ah)e)z(XvxBl7{=;SR#FCXo)%dh--L++!K9&iHs&D(B#(NV?Z$V z@t%Z4(;#7!j<8=n+bv^Z!i^qUPSUZeYe7s>!hUClVva;fZqAUt-0iL#QK$PN9s2#A ztvox2@)!JB{^_0nY98|4DQ$oXG{IHI+9W%dUC7Io25imV@AmN3w70(jp{Ft;?- z@{!w$G=027r+ccaO(6O68Uahhm8u|a!Zo6M%$znEa2?DB=`)*OtUoSM?ER`cllHW&nD-li=9^6Fv z*<0NW77AS~z0Ta|HEXbpddJ4xhy1~*btkOvz~-UsWjJ!U^U3#jT>0(ijLp4x1&wY z32^Ehmf$x77J2!H*=hy}_nu;saiO0Wn6@h>HD7Q~EF|42J2*RcH7v*|vhI={C-b!? z^&Oe^>woH4WJ|r9$6C)Zrnn<#l>$ngdngq6{%1JZBdJ%O_pr()-oKCkb*bjY_X&0N zwN#io9$|`Ur+R9(Vix(`C$r7!J9flPb*jo=g}|O%U9SJ)EkOy#6Te|JiD(b(4#1@y zZ|L<4o+hXv8GCc@h32fx9^AX8;OlqeVxF*V&ynBR>F1#=N*ZoSzx%i(ulGIH;rI4k z4j-R3bct#aKa>(#(e%5aa9Vj?@qdQy5$!eqS6t{v32MO!I(HOWOa}Cc@dv2A;|%^CixC@}^=>CfMq4AoTc_hS^vd<+ z4al-1-g&DhipsGWd17er9~7!w{jU%5`}_$1L3i%z1iAhR9?5o=R!O`kL8I6*4N$L4 zISo9SgM^@zgN*O6BziZPnStTS-L#YXG)GS}cwmjGTNmTpMkt<3$YMBuZ7Joxc13Hh z>OX5=4hNZ%YmY*q86Q;OOge^g^NW;%TbrQaf8v-CyRh&890OBOScNi@>dCX^qN6dr z2P*XV5ptnbAYZ~+sqPQ0g{3MY=Z&mhWx`b@C{{peN3*0a9@M6*X29{nc<_hEftFmgb3#L#m=M5Dy1)U%DK;5+>Re_6DQI%7?LBs z{qMh>->RnQ?ctbzYH4bknw16jvNXGDfioH;bQQ4ZgzSuUpSyyD1N%bRv)epk%W@;1 zU>*Qnd|?2f$0!%YEJjD-JKA%yJniE{`y}IKFvVeNt@xNU3#l)h=YNT;?O1$Lp#!Bg zsBUPMm6wYgcsFDMhBvk32tDhi^eo?z&z=NN~rh%lb1MJTJ zfq{`UEq*3r^tGyNK4BeOKrIGAGuPJGcp9v8NHHs@0Bp{;48QxlST+tl1CT}~7FBM8 z6qVOtDMM_z!wV`l498}TF^J=}c#A5BRy~P}&D6>E#-m4H5>yKV>|RQ~_ozlGsi_4) zR{{S5A5n{w<(+3q0RdipC3t;zD;qRj_#9MtxDri4US6aS^6sJoXeEbYc>YER90=d; zYIs1RP@TPg{Ui=(O0VZMR)NH&8j z_gQua6jN29hkE#6JE+(#$DC8%Pt|ojzkHyKeD}otsmHPEQsCy#FZ4#}n;h9LDZkfl zv5@%okNpG(oZiBBumCE#z)2o`#EHhU2elJF9B22pJ_x$ef!i4m9uV1jRm~y4h{s`w zV@03(ZD+L!)f~@ivm`M*6urx;)v))&-xw`uHTd{w0nS2Z?stMoYyLZEVRj67@jpbb z%2hRMX>RVfC)>vlA9{}FlC$x{SfA9#G_k!`FwUlVu|xGJwRGr6tVD>bIrf=H`ZPoRqD5X5QyNpO$e< zHk&?@59e1wpz4VM(@qjA{@@^z(ya`S{d``)P zd|uhOUTu>%>7hU2f#T3UKs4oBF5f?y)Y+TcmsT^<4q@0kaPd@#-5s{e$<7Y4w-?`; zbE@8CP{P4+ox;~p0&W+@1&nRIU}I(dN*C($2KfQNjL$@SJ;k`~msY`PKIZ7mNGkb1J{L#e&)ckhJ6CmP1Mb$<_R<}h4W)gso>eXDDwAXGU(_mGYc$^F$;&| z^!Ztw%Io`U5LZ>#x>*$4StlvMfnHtNK-)R0x!<164YDzSM?Z7P9)wDix}KCcMWo0S z?;2QK`b_ywU7oWoJKpg+iAXzv3Pg^Rp6Z1_+^D>SGP&9t8w#q5M#R>fHUxo|ed7O_)r{@~sg+rEnIkyG>T3u820@8zEc=rQ(K zYf9{=o@9p5krn&2h)mJ4_ymQr`=1-rq@tMZjzZ17NyRMRG^Yhl>|b+e**zS1TNYSk z58{>mk3ECM+$}#7p1+Bl}~T0 zHV7Lo-Ac+Fw12kloJgRn3v%@~XwfBrZ-kvG=CCdm^w_zY80T_uNWtiycja z>gR7studWw)%N6?nzgMyG4IZcQnd-S`x%QVvDRn{qsJ0Pl60M$_$icuU7)ZoeH*6Z zv4sWUT9G;Y3TH$Ms6t*?RdsF&Q@X4hFoqkM^SJ#p6w0E2P22%zfBK&Ow_f1@TuvL2*f=+qH(q$0n7+-1IKOjJ_XSy;P6N2pTli zlrmE@@6gIB1fH3sTv_J`;0>@_JEL2NzpqPaZ&u<&xI&Q?HR?EIwMk83r%z89Xt%TZ z&WpGIr`R}$AVy_0GBftT>TLmB+CE8H?Rfj)WF?zZm-)``X$;Hqim1Zgim0CSdyOii zBt7R>4SR~eT47iE9{uXB4vJjl8FzEXtUKMn4~meJO9zmooMpX zWUI9;iSJYaDU^xfdqC=?M>W+f(YbGB=redvDd&NkB9!BvY{P)3fni$bL_caVg;Bq( zQY*7%*)fw|vfp-6)T?$!Sa${O*v`F2IJwP@(Z*hWWQv~O)XNx|A6f`faZz_qk}B^r zMjt($Ol2BS-+E)t%Yj?^gPtueY>HLnUlc6kjd|~dU=aM2gAA%covAsT<~m>6a?mBi z6`YlLIe^!Yx51_5pLl%i^ku60pjnlD(9{Kf-}EO%`+9z$&$(2`Ue91Qk)|&b zeph(I!J?&W>VsyQ#)O>t;E+HuvtfOJUFm;9$r9tNZQ0k%2jJ)OT;yfb=sG90FlO7r zkvO%ef8zS5XIt9>Z6=~McC&q(RDpNJQq8z0nu>DLJ=HL|T>nX(Q{(lbhluNP0asZk z-P85zDSem(E2DVJjEZ?59{w04+E3#9G~}YSapSqYyo*{W(MxoJR+6qV(L3SpfUbTK z?skn|bN@TOGHQ$%?In2;qOVlaF~5u)LcSIB4K(awho}2e-={Ee5t>!09;J(Abf+f@ z_(je%Ryqz?@obz#679%em{bzmD@ag?)Zw~&LUz?z>z%9xVp~Lm`})hYNB(Vg1!bk$ z$WGBokAm`|0Gu{Mck9pKl5-<@hm|2=H*|$~g!k+o*-crd-29l#P3+Q~^Rxu%`RDEV z-*F0}>mSDUC63oW3y)pG=P|*KuHwpna^`pBXq}B)FgK=I>+^NN-O`tG&+ZpUjSNm$ zpwB!vT2P&*pZ*hAV8MVbDlcPMD}N^ojj4Z&V#?zmBtKU5!qB5$bC!0StM9ru)i_|T-#egHwQLmg+tA5Y-crlp z&gyU~KgMgnJMJsmYyz9X#*32J@BAE+l~cJv2pb_(lfSt5|2?3vY#SIWmK^1)U-{b( z)RaiQm8lCDoW*3`|@iHZOYU=^0b&2SnSgj4=Sy@+BGtzZp;b*Y+e=eQ5pn9&T5bq#?Yq31 zAq?bpT>2@UZV{`|Ve@@F7%I`M=T=&L17^y!UKr*;)g#sonka!mQ{)#3DReq%mIHa# zb)4+ellYrX zBKbW9%fh+vIf2i<9Xxcemk)G90PDhxw$KMY#*s8Xj-R zaMK(m!y9l@@eXg?-X-OSgA`_=u}-12xsdi`I2+2zO}+-;vWJ_tr9MZtEw3H_IEhgb z$N37fYTosF`W~4)aQovEnZhreli`ER2zB|)TlZKV{4Pjc!X>YGRD^N$X+h_J2H{u6 z6C9zmQN0SOQCRXy1h2H(#rGzMzyt_Kmx_x8;IK1{DDiE0qQj8M zdd6 zBdW4{wMVF6{>{isilpBBdZSSH65(q2fz+y`9na%Yp=VO}FO?yaiRNxPu$}YvZ@v4) z^7pTJnbS(XI=n(9y&f0JWsP?O3Y@3&(hpP(_%S}LM|=-sI_-Q%Imd0vke3765OiaU zaG3SEmsQEoBU{F->y5WveL#scI&U zt*)uR3S=O(W!Cc!4KCYqGu@Kp_Ln!|JXj~f_FAB4D@4vrZQdh9K!mcfbe9R_rRCs| ztFk@&5`8V25A&dFXvm^S>-(6HGM$<$t#NIwAK6Sh_3`O~mT-#MV$_Xto-D|IwU-%7 zxZ)2hyVTqFbSu zLmD){#!7pf^0*@r96n*|?Ea;&)bvb)=1A94qf-QmN4t&CC5&RW=p}_o9k^tiMX3XL z?;18l7XD~$mCaJ4m~PmUjAXLlovcZ{b6=AQVz;b(2&2Muef|@77C3B;iF~zI*&v5N4J0H?YK!4 zp2~wUIY0BY`EC)5WKlj&Iyvg~FUB0FX~`NN;}OQ-UmuX!>FLsNLXr6)j`{C*ABTPt z?d|PX#yPIygD$*&>OIVd%1Y7qwO1cCM1FXZX7s^ZXf{BuLR6z@amzkpi)^;`Z6MjK zv7G2C8Pj5lvjsL-J^F|oaOCN4jKC!e~+ z)0SQ<8;0PLY`pxdxzi9MbR7;b4Gd%#4+{;&#C{$tnpp_nat+A(wGsgEE?@pJ*Y7-3 z_zcC*1cqdJoV5Jf76SWn)+e6(JD+W=*(QZa*CCu_f3%@3h!;TQ$W6M)!R(2WW_XIP zK!eEeiEUC=$YCK&hx0S-WTu&o_Ieie{epW!SAPhxwX5+m0H?8YKib^<7+8>B{4s6b zR)~JD%yaQ8{RGPAmQAbBLCmj8K5^8o4KB7%u9hg6?~p1SoRiBPDhgS7sWOr0 z`9$2ma=C3b(n>xEiP%uPu0&ps-d_z{pFiycYvYk^JKYCO{yG0ATWm~(<)ZpG>B;R> z$V^DO3EQi1$g~q8+fv)!@y){U07O$mK*1{eOdnzLoLY^qVM4DEFC0X$+$gdBdsPTx zXe-a2hSB)YC&b0hPH7wr=XP54j0*90)TyaCPSpFt`8qH_rdBb#=DlRS&meiBc>@*0 z;hOi_qFq@~?16%mhUVjzo6B;AXtc?uj)sGmpcuE=t2W>%tvi+D*$ zvj?68lk6JN@Y+n<^1#^+vWM~UY>=)*>&QIuH)2VA>O8ko@@_W+M?}=onIJpEP`o>-A z%bqM+oE3GP{&>ws+{ql>)Z|=FfC}bqi$tsj>P}Yfxlz;dVY1j*%BbY~4jhKfFm4+z z_MReXWY&|iM3{! z&Fj73{V5TN@LVj8M4!XY0xdTtzWje@A#`PdzFr374yt{mUP_%3+{0IGFuM_ zZ3^;BCp35(RhiJ*td(W`szFmr6Y_p5z>P|3SeIAn0 z$jHc&Jy!2IvYcRUqtw#QF;LwvmUa{m=b~8ss zQrUw2{f6i#dTcR$G!U7GWDQf5~}yYDNB$v7}I3Wa9zscsh5Nyc7a5!RzyGMS&z z5X3~6&0c#rdFVPGhA?)KjA#+NtHrt$!X)@fiijzPUw5O^+=)?*sLl#)mT;--tJEv! z6ZC7Zb^AZK6scDi(w|sBz_^ za~jy#R*al=@(Q#zR%jj-Z-|8B^%ZXmF~NeE^nDR6@iDB+wMn$Cd38n0PGO8%#@m|p zOmf*zb_E=)s&D)Y(B>{%^$P&^3uG6zBL@4eJ7fqqCa^-h0EGSu?G-j`*z4D}dUzc5 z)Yj4bxFr&w%o`MD$d&=egRS!aK@@+8!$ynkK}w9WzL4Khrj-eF+) z7)!}u-x7a-+WFZ#i~JZqr}I8I-tw!}gfI%NHY51s$-OWeff^?j(+w`6C6X{9Z$J`( zrmZsWyFOmKJ_Hqq+a)!|>c>4P;tser!5Xj@zOX?1*ElJ!HAFsMt`WPj7ivGSTvp=w z%OYGo-z2qs$kYJS=JiVO$qHlf3r++5Tk^fZzam+-)!wTViUsnH1AAGNTcmWEtQ57{ ztFymX(!wAT>hNMxb{o6BU8=!by6o?t-xadAf`K$P^M~mD8G)D}T!E=lf{^yFv`W26 zS&X8l_A7m@%~2|3St}3OtY%{p@-k1Bol$_Dqiu~r)BQ7@2!~II!i0gd+s|#0wX#ot zbqWk;aZHw}Ge(x{Sf~8Vcr|ll|9wUDJIZyQ+nc>h%17(wXfl!e_xn&2cTU3=i)E_i zb~!7|D|SRat8N#;bZ1C2r|7B8eLcENE$+@}L5w}&!bZr)<6Z(I8Oi@c)?3F#88z+W z7>EiYh%^d{fOL0Rl$3ygwB*vbEZvGoce4u!C>_EQ(%rB$NJ}o=oxijAJn#E{KR^E= z7SDa|Gjrz5nVIVv#a&qC{oFyf93vM^{f`h_4URt#ahep!G zi;QDeA?;Y#7r%~72pwbgT!MFNzGUGi4bRvWvO4?~W-BxRR&I551U>+!PJV9l>~tFV zfYPg&(uudhyO(D@m)f(Tpysf|;TQ;0KIeQ%N_^Ic*pF|{Qn_Q)pm?GAl7D1G;`Qhd z)AMeNvmOS){!#say@JSM!`Ev>zm`Rru?6!;kdobBC35`d8D-3wmyRD!ob#g8GA1Pz zgWgQkmRu);EV@ns+zSsGy7wCgtvm>fW($`Mzw(5OdYDwCW-Ct)XJ=*;B%&s|QvS4z zM%JhOmR9w&7NnDQF~6BlJT3rAz3=-Z0Lss=f2Cz)?jZV}UvD{Xqs%jjM3XMa3gMFP zRLkN7RL1y59S{rC#yV}Vk#=~}!8lE5BXL1S zUJ)tnk|D}({5{{m-0sE8>d=AZw=zp1hTxLP%#176{PK1%%yh2g`cxtE3-HpsiMdxN zV+lLXkrls^W;UwY=XhV)UYllIz)|s36x%$9HFTM@OAxU%WkRRwL_@! zldQv|_L)F{RHmfC9a}}KKgbciExr4vE{&QcxqUrs!LcLseUELwPyv4?)FcGSpB&py zgR+i%l#RI4sxtLzuIq=r?ugAKFtL4?lD{v5AIw$FXwiWE_~LgEvfIQE-THg2sB`Jk z^L$mWpkxHoaa=Rl&J~f@XM&)!SGA1n?CIRfn{y~%xin;`96M`!@wEpg#1Jgz(o8!Y z@~wsEXliu|Z>09DozK$(n-7HtY5{P;p2;Oclik&uqni0hiKao*`i6oMOVEMWG6lM^ z8s(&mUlNXY-JLn?$y3qft{=O7+_US}`*LsAt%#7F?cZY=N4m(}rK)`5Uw`(b5aG-X zu$hySR#XJkw4r8mRq;#lx=A6L{b_!+86@W|&N_!84WLiMH4p+dvJSzkwWv6IOdbTYYs~H@a^pD!K;Y8i+AKZ5l#e`5^ zJ?x#BSmVby7_glgHgtzgW_zYho;mDok6&h*Qr&1&%%6dGyrM$i!|vMCp5NWM9HgBP zd@F+^M3IvrGyjbJ(j?}+X;vbkTu;vyPxbn2rxs@F@0a@v5r4impDSQCHssU_!?m6xkyUCk?n3`^pgmifr08)NW zZit`F+_z-)=tKY63T(~d>#p8sW~-dEDfxWE61GcC04&?%4W3F{tHG`K2Y`#8l z-sq^}s;ynDtW=!)8?d~KK9z;E>7QB|NEOircl{~H#S&}tr*W2pKv@DGPtXvnik&24 z1=0m%2be7^df~EiG2gT;Qs|_x_|s#Y&cVD{db#R&m`y#&Ju+Tjt_{52zPsqdbB)na zWv9bwz+@Qi!g2yO&9DZ2WIb3EsT-gkgg%n7p`|>}BgUEL!0y?!wD_=AlJMK7!lQ7L z?a4`bBdj%BTM0q(SCNhVDT_M-e9f@c4L9E(&s>|zT}XW!Rl~(w23Z8>G$zcHATx-@ zl#zt^Y0}7(I|b}oSSD%@mo0!v)l`MqAR9NbWIR*c`Uu(mol6*`R9wAjHiEfd19G#! z4{^332nOSmS?lJU@jo0O41~Xvgo9BqfgtYJIy#dQo~tBk{}J3n3Uqg_M!i9b?aIoGLhx}#ng|0VZh5DcB{(Ma`%og~Wba&=8!Q19}(AVo%v(i6k{ z4>M&KPwMo8cNVBu&02lc@0>)7pd2C#=kC|yyBQo$pJ}1EyjfXV<=J$AyE8YvIekJ* zX(9N#&f!R!?6&T1cn9E7;6M=)*W5T~<063y@FYIsKp~1zOwmjrzi9Ah_mfU{!~TY_ z=*BU}6K`E`N61vCk7enlIJh&p><)){r^U;VcG2^@PEl zJ%80&nso1(tU|%`A6efNQ&Y@c@VZ=+aW&TQFQul8SXZ5Nbj>Z;@aAjhre1<)1jLk+S9CJGbprPI}bDAog;# zYO4WPZl<@FU^jW(x-!`MeCEDq+r!ebGH#q>klK1%7BP1Q?s#48?N83PL8wNtv!G&y z>7-J+6l#Kk>9jb@)Ipd#YxplaRQuUtQ}YT>TBjlEhKdbV7u^H8O0J!xt`3*hrO%|^ z>Gq!2vgbwnynYw6N!(rL89fLwkz>CDL5ZIGjoiLmz^(|)uIvWV4^P{4vOlhMwi6U* z_eKyipv1PZT6<72i(H@^!Ff#L6V2fdhWr$5AA>}mkH6`8Xu_&!^h-lxi0;`R*XW7E zd+G1AwNO+rA^t>bR`E1#!mf?+&yHGt)^qy}9t(3-LnqBge%9~PWwiNDY;OErScO+Q~#rfM>N3?YMxy$J(CyW76`5`dD2c_f=l{KloUQQY4yKLdb7Z^(_HZTNwL#4q}}W; z;H)w|1~B8f$~lnRcGi5t%p8d1t?T$aH#euQ4aG(-4||5OF6`&1X^v^k1ZfQB%hGW!~08hkI#L%#m(QK)C68mzCu@5sf{EkT=@uvQ(Ci6`HwfX}+ z)MwVcrbiGdTdz%kvT+&>-*O2ID=>CQR;L{0^(No$=S_$qVIA)f;AX@JfGteUGi~Lc zCA|rFrKl(<0;-^rmz2jB{2jr&5;mW~X2eS2M*+9UGI+eV_XR&DS5kb|LTLjsG9Wt; zyPOWdwuwkX;7?DU#P_(h7!)2+pgY+ra{=+zQD@&_3v>Om?=(t#la#J{r?f~OYVYxN z6GbYLs0G;18N42N(vjZg_6n+9Y5flY5yGe&`Ln!6gZ*~i<@u#^_l^fx{05upzugT= zN=wuEJ1xMQp*<~^OO#31dSpT*N^aY5t%l?LsrIsY0zh1AbT=EV5beDo&fkkB z`u-le*xVOvU!a~(2r314IJYvcfm{T$8P&3%`p;fi2Cu3!3;Nh$)8!C%_c3h@v?`Xd zU3T|J4Y?4z-fAvaBr5V{5!xyTi4f?WakQlV*rcVkc>Gs22XDL^T<^=uV9Hb&r$cHj z$=BwesQsWD6N1ygRUN{(s&1E*7+;oI3lmFx!hyQn*SnP9-N00Qlm^jM(`xP5#_Ws- zK`+pr7m$i&{@Ua2zM-&btL7-|O+w*T68t&CvJg7h@AT@c#xoC2qi_~uRtPLG}3$0oxSwEbbg-?&t5Eu5m zV^7!@tugr>Dk?7w;vn0+bCw7P_)Gev)zt*XHUkT51p@-*;KU{8jWecLeSY#$jg0m_ z=@f9pQOhfVki#aLu z5oOPkC5?i(6T=6!8}ExW$-5z=Zz7G}X{MsBXEkd*RHb9%?slc_+BFFa^Y&yp=OfU< zvsN|)P8-D~P9A@Rd)Lc}qZ&1Z{6%l%0H1};?+q%T{cZUaA-ns;@}2zy9@JORA!d<{ z3*$V@&#)NZ0YxQMCAIdrIsegn`XHFszgv-CFgN*8A&gUeulF@COHonLpNK<1cy>P; zYO?916lXRCyta^ii*mjQk68d4K z2};%Pzg6h4LFF}i@?)8)&rzCze+v_H%@oQzSOR_I$}=mzl8{WDAJ!QOT6aK+5Lf?9#*sHJm7{WA0 zF$!57sYS*@eyEgEdLBn&`InB0olFdb-aPJ?fF}J&u5%|4Te)bj#<&;B=o)C95x1T@nqFZ7Ne5;j;2gb zJ{k3n_dR!VbX4)0X2&7h-jPJRDVkK0L#{8^EWaRa5ex}NPFjaHTpR)tJWOO(>qv_~ zvi%|Xe;hRcU3&${w7h&&ftT$?KnCF=xi@v@v7(Y(G-?UrA8fIKqCUr|fK~-ql$Ms3 zzXd)HZgE!ceTk+I@@SuTYn6=Y%7l#AKX##Yn$vdc>9#Ym>yEsfRhJn#KhqnfGYLjp z5D4eJ0bbVUq!*Xtm8Q;fFhwXzd-#3l!Ll>3x=BCe2uEU&u{cHg^YNNR8pPqYIzO?97z3=vIIgL?Bw5hB)vw&-%yWOQ$ zK}$Y#G1uTG&?hSEySTWcuP3A_&;eDPLTd9yPDZ}(GYzwcROQ0OW#kuoN<7Z9hn$iW zX5o31WZMy8{vx4q9=ZIh=VGWR=GsObTQ2fe7xn~BSRSEG|7ZM>4wk%~{j(N9)Rm+3 zN9LJ6g0odHVt!$vkQDU=WQjta0UBe3y+wmL=5k+i8T@1p5gwj0WS$}mObAFvhyzB| z*j!)0Oc2%utpUXmOX;9iZs_J$8TU?36rf3|*TePDZOLw`XSMgpDU75+qW_du@3F8_ zJ3-1)^Eh8*o>s<+hN!84b!MS-oj%ICY1TM;9PW{$kh5ZTsPOv&Eqs=}p00Nu6_f0G zqEyQ$a62`77NJ$Mq%*X&Q2>p(xhE&Nv1*`LWPMSPEN`6AHcnAYM2jEjymwSN^`qa@ z`E#}52=k>pzy0}w1WJEYJsmRADW~6)H0`QRaduwdrYH#n=OGvu-{(^3rDOU&SMzTF zx&zXkhhbd?j~os9L7~202Yv@1sw-6D6>g-gX)l|4|Oto+nR5pubAU^?70$djj{>G z9!MH$Y4MHP%&SiAlaQ*yIpmQK$`o;Y!JF>r$ZM?#l9AumPp>j5GVboxFnck!t1@9? zxBt35EdfwAc6LaQ`_+}w0P$0K8Mrzie@E>+9}bf68A&by)lkIRI{E`7h^(Z&uu_9eq+P>*#OXSf;)2X?NbxBVrPsA_lM`g8D)Q;9E&T@WmNqWPwUw~3g(cWAZ|=#YBkV3Q zm$hmCKe&;G9`?sZE)OcJ_CAlF=RcSJMY*?@Kp*|3T6J=|d#Zv%aAgv@-{94?XBD84 zX0n+H-}%j|Z%ioMjFZ(e6=)R`n{1+WLj{GR>fbrzMr71Y1t!`&Qv+ubbwP6&RBd>e z1#b`j<3UFAEDxNODcU1|q2RfiCR%y{%U1Ex^Fq4W!l z$y58M&lf=j>9w1BAP;piefzLh$r=I3wM`(96loyZH*Ep{`MhtrUa3|BqIRV#?tmZ9 zn+Js{O_@CLS9F^N0e?USK~1Tpz-+v^8?(p28Onpc#YTYw1Myl=Yquav6Ky0 zfcgb|{SyFce{(NBChTD!i!sSiXPqwld(2B-`?t7&japvlHxubD|D~F3zO9E*9X}Nb zH@XekJi6zG_keI5iJpY!1@hv^b5E2pUG9gM$-&-MfB0L>p0OLlJNzs1<$&`9!Z+vC zQ{3PO(n^V6B`o?hnBA4z_v~bP*>Fg7H}3FmvOtrw*~zrakWEc^90v*jysmjvkr1_L z*ew=o(Ce%+#sPK{58ayzuLp*+MW#Q91FF?T}M;r93r6$nd;Z94J|I*XY-9 zBh6xOf1Kj2J3T;eXrc)-q@9Z3*2P(ID34UjD3=o+BkIByIn(5ZG*yz!FY~{bm-?#= zoCo~Bry#MkiGt%T+M}6+{f8*`P+7qSijkkn!6iTJ_oyjP{=QJlSLl?Ui2T^NrlS5^ zYo`JO`s*1={b`wx5@-txKHY5*eE)_0f077%sQCDJBe|R6z|*C9)lyzvoyhtLV84pf zBYu4l1C^Nr34n+I7S`enqHHzJf=fndKn&+_3`}Q!^}*QXuII)5%CIw;jAg4o0zLco zh1g3*F8!LkyVj?*ERJqr)&7yO-h;Vrd%~7^a;YnYo18?5JNGVUxH8)p%;xw34Am%gfGpv1ZLK-?TtG-3-`_AOAdya8 zs^n+8BdS5KtpJ&NG}1305MqLog>5WLx$Oqu^q#bi{h9!{RQ}e^08<i)~F%a^Y-T z-Z=kgqhiegscjO|nP{?UVCPV+Yqx0kyQe9A*G)ZUexXjFKmv40@M`MTGDgTC#Gtx7 zl`j}noe@!!qa%zO7K84yllb*}BD=fe#J&p&GiDL^cE570ESp?EDMpUYmpe);bU9X4 z9}}`WZ(8uC`|GQ!+)Ktwp@k(1wrKoR_BrDY!_$5}dulKuK(uGl3YQq*KwS&sHoJUc z7GCgsL1WsDp4bEfKjvkrB-c{usAU)}*!%z+08oS) zrR%ECA6{buq>uxaK1QAgm$%*008EgQEAfWXk+fg&T5~MX2hAVK?Unn4tl~vHc?u(~FmU z`^wR{-brJ_bY~|2qsOevCr2W3wumRDng%_UmGGyY4TZ#10G{Idxc;=}IX^1CI|5Yl z^Gq?ti@B)va)~K2@5!;(3E*&`W@y@+2Y|k%rKORIYJ$qE5!?(pwH{$;}G>OZ`^TjNI#*{cP>ipG~=yXmejE@8hxKg}0!f|2b;28vpk zW|}vs1G%Y>LDMC?76BsQf^7muau5-)%=?N0eE9$wwdl7T>b_>kUbXc_y4c+Q_dQDV z!SrPZ^9P#?N?+&hLIU9&$#Gf03MXVWysdKQI5+Ee(#?*;X}2CE|u|3(NoiivAd`cwhUN0P0mpgT)jK= zq9xWM$t4VN2c6Sn3FXUs%|>@~*`u*ldy*$ClWiy%RByXT|gp+t)6E_G3V%FfAQ-^$R&@AA% zB{KbVktEM1cE_E0-qB@uh^m0FK|snMim$sRF(p7-8Vyx=`}V=@o1dOy_6NX|1Pz(f zO5{0ggc=AxX=*)Fsb+*ttj4%jdI)LpcYx&cKvO+zfod zHt?N&lc`eUZT^Cmk`7Lza5x6p z?ta~cuutJU5`|+tOuE_Rh+VkbTwcU-hZu1djWYZu)7HZHX)G8whgF0zgnJM_tn{C};03ra zZ9`H21q3CtPsbQVEkD_WLPmCq^$^Q)%dN87aXx(Sml4UXPmogSms%PW_ky@n00rz( zQ#a-Y4K5R_JMNt32wXM#n73&4pIqR4{k9^$iH;j+hGO2nf!K9b*n0?&8`;~4-ER>* zdV@*cuYMC9xc)wH7;7jbi5OGR4~owFK%&{ddnQ8#RM`qzaiP|Ul$W$rAjQ;Gzi4JC zB~(1vCtK)`TFanQUVptYz{E%>=1R`RSBY!1^I@^NTc{A)~@!naqTVC1kl6Z$xsg_F2K!x?d39i^$JRuF-L zyB^_+l-FcywxHUn+v+|pG~6v&fXQUc1h^)-F0K#9;@=S``8pO)j z^(N9jjD+Z@B(3%QkT2?7PK}LrXG;H}-!$MocrVAS#IX~S>oC$WuJT0$RZoV#_fKPc zDbRpJHszgPIN(UY!0={UK@GCCEbu#oG6Y^cfYhc?k^@=FNaN!ES?Cx;c3lHtJW*>G zyRRy<@u%9Fso4dzPgt3}pa=WzfTe&W(0KRi&89%5_%F30t^ zyrj@_fCn`Z4ikPvfO;AEyH|UTfCBQ&8dH8ehxek294&w0#F_vyPai)j*yg8rAf;sG zW1?evnTX4pg!R?Mh_>ab=8aaXv#${%$)_Doo&E0ydg+Rpc1sGP>Y!{|@Oi zA2)`Uk1Y* z;8D7F`};e3n2B<*3-SvI=z@?1(5b3_yC)h-&r0+^8YJH0TzzF3_}WDpm8=vUzbBY6 zrVhkE16#tfIX5@2FBlL0%;0o3)wSe_agXV+q9V*EWYYRUAUXjqumbdP9uq;fX%VpL zKa?nkT9XaUJ9e18Kge=9#^vF~XtOH2e7QxD(7p){gEpzP%or6L8Jgf~mxP3d+U%J2 zSt86XViM}hy;x%oCHo%wgQAk+Eda=BzA;X}=xpmPX#=ZJn|ugCQ&SVL*yj5KuP%!G zuFwM2KEUB|qMm5{+ZfMU%FD_~(=Z*r|Ct5+q0o`kQGrO9p+Y~0Fwt}0mbVTjjX@a+ z#Wd5_y#weahw?R9*^^~PQJek|0ll_TCBp}SrW+=ky^bEbc0xD6ZX+kOY7u-^v*}Hy zEw}|Rn1xRhWnH7?W^y~ox*{i|ap$Ng<`5APsmfV=p2fhdKVoKNmwY38Et@ygEY7@< zc?e$}wGuRiisIqUn;I6+5f)F|1&|0XI9(){X!oR?63Yi&cElIxfd%3<-4P~i!*7NF zRDGYDF#X*B)iPJbH+6FwA32FxlbUHjM&wL$5Gx{nOh|`cIw$Eyhn@q zm2m5*fg^|=X}^2-EB?w(?+O_eCsye+)SBWliWeA}@Utg~RK)Xira`bPZOfTr3=w9F z!G^qF0>p+uePuyU*eAl4cz`~CeH$ASDA_du&0t>rkA)^y$bfPC!wuU6*kyI|Aoiy} zw>fKx=c5<)%2_f#DhSdsGn!oRFcTv>5>O5yAh?hOO3-Nm(sZA>@)11|7?AsEHmJYm zHfySY<}d9|7{a5R-4;gcK}I-LaY0KwOHKM|q2{k^b6t^?nVlxFo6Lw*ba`G7_p;mT z=w>mHLdp@0e4A*zFxB#9S}0>hVy~Rz-r64x#dP)=guhBTKM}FuEZ>}j9SeiFB zM|9uq`TK5syfL4#OWLrwrt#wz1sXdP8bd;Y*(S859Rj{#A9zs5iMBm??7{JjG5|&r zgQLyyM_?yETd%p#4IG_D?CG}Weq#lq695)m{00e1<@#yTGseWS7A)Nh3?z|8TII+3 zf7@pDleMYrcDSqp_M)>NCCsroI;UI?<*_)*yect3$XHCHz<6$$L0%0_ZKJrvlvYle zXL8y1m>{5bwBn}*0$#0M{;Q6o5z?`*BH}`C;)5XRVD2VkdF1rawe3%jg-Yg!rC_|uZM*XvfeuZAd|q^ z?CfD)$0rSfO;AEm9UcHzV(gYwWo$cU1fXd=yF~y@t0&SuD6Y)g`7q*}`dAKtRGX`B z7#Wm!v02N2lPXU28l~i*od4(|%UT5W^SMad{&&7#lW^AILw(#S?Hw{R?Ltb*c?6j4 zeeH@gA47OISCtjFxnWmu{@TR1{(ENb#}WQcqqVy{cl-w|O-S!5NBS&h#gq%yM#r{x z1qOEFyWESkNF2Em>PG~vB|@egeWQE9Bd5vmWa@cBL>K&AObP&lrWrFTk-;#mxZ_IRpN{C59)j)~KQ})C@4*+Y;7f<6kSy#SCf>SA9lOSBqklbok zymk@GQ+305>Z}h$P2m4oVhqp$2owuAecy6(2U%h(*K#I&J7+#nQ4*W{fHDDK51Gl4 zrHs5e8{Qu7fJ*SIYph1;f*8DyVytfU1KMMlne8h?Vu{d@WQ9k76#fDyiUO!QD=7i$HA#t z!~V{lE3}c~z=7Jc?s%IWoOPk`h#TTLDqwMCw=ge=F@fMsQ*h1nFSKC;+OMmNX_A#% zEPMonWtDV(X6MjVI$5OCcXqsef6pGF-PN%7-VEAjTN6^DsbR(mdGa#9PYmXwA&KK= zhj~5BZf5nrVeGb3rA*np#+V8Za(sKtcyD%(UZ(iim<%o50w8H$6r2?*gU)=xbDDul z{=haVc`MVcQ$7WM3Q8UkeB=zI+-C2AC)95$a}JE4=(X`b@aBQ}zm>M(t^Zf42)bLG zSz3a)L6iwt6Q-isJ^`Cuf(Bp)*aCoI)*{_(!0b)H-T=HgT8#Do&x_ywCkxC`23!hY zdB6ymF@i8WBa&Bti;3O?SIG;2vBhplP};OOe-~FvNFJK4Bj7~M&bt;GA$p1%1X{KH z200B8tv@+X+az?N;I~A&zR-{yf!~#P<}mIS#x9uZo4HX1@J$8n{(X_lBMv8S5uBO@ zh2CT*SY6iM(2z2M)(~h?7^vmYiaKC2T)MFZidC18vy4;YH~zbU2t)zu<$3@g>J>i^ z8FP1!zN3L386-TS_t9%=ukv*+fg%CiaI`l80DK&~D$k6=od#+TkGR zy?<}?qWF7q(wEd=o6x_N60)ZfAjOUI;PrdOfnOG4z`vRwh7t9Hxf_-0P=rJR4bfH?_;ffsaEq-WL zeQ1IY2FiJq;7uEdK2T-6a7l{%dKE z<0RK9psM&~kOdeEc>h?b*s0KUxK|Ba8Cp)&x5#tc=o+wsTT!#ehK>4I-hdz2tc7xAFNpl!Hm{_ z^WPV>?{ai_G1Brf6Am{hJZ3bOC1Y-RN#&e2WLeaz4Qk{kfq7^v67ahijXRTY9&ZxS ztrYN^{}v~gQuzZ}RaQoRuAbXkKPiv9OOJ=K=2!B>&uD(y(ebq~c%S!%A@mFPYx};N zl0k)!-4i3otUQoRv~C)?t=kkS%;_ZZKLrr^hfF40cU$b%IufJUKN#%Y_?72D0>+gZ z&=70k0)t{SS1CY=cUeb=xwPu%j@z+H4P=<&sDl*9)AKeDL3y==*SKj|sPc-BefF zXk5m+mq?n}qWMLsO5vl)iJM~CfO&8yiGXN|g7|RYwh@n%$I16s6%S*(6)k>&dU?)q zo`ku{o%p0>pQAXfj`|t968WxLRjy!lL32aJ#|1y63Lj<^DYATMQ`nHAChZQlFH&q9 zF(}5A0iTAMKL_o-2**`cm`ir<59!Ha^SDIls1 zPW~2_g`3mfHBg^B-E@dTkPA+;>BjLlyYCLbA62{Q==!dGT3}g&@ieXas4%R)f#9G4 z8BOPU$mSy2uH4!7wQB83li*Cv7sjd+t;dzS^hDVvxRvwgNTEpH3w1M<9K1=^)YO4S z8Jw_bL8hn6i3bf`DfuQdi2f~%xxtzXlEr$Veb0-PWW3>vv4U|-r`&@=B^IuLuaZyh zU3>bl&{nj)ggCIxp0CqrJ|U8MlLtPz&8BhvBT~>_6RBMKjD+QOI{G-0;^M1|%LI{% zAAt`#LLsw#A#uj+t6SciOclmGS)ssznX%IDo&{U>`1JtnKxwO4=3tR>{a5MKu`UL3 zUvA6y$_hOhQta3pxZ9Z9@jQ{=BF2&g4ii4CHG!*C8-ccQkf;{Ma{Y**v)=2kr_N%u zzRM|2EzX-qmQ^W3`rwc9ttKvo)=AgwsvoOkjFTUpX5InlX#g624w&-%#vJi4Ca#A2SZT;X zrjR5O|LHSm8SC5l*#gY%qpJZQ+=2~Ift{=U8nwLZ)3EMfb5WS9W&t{W3|yxF{Hei0 zqU{Mrp%KHFWvpL$Pys4H$A51nc{K!>8LS-}^y(a4;DfJ&9$lW7hn}9J#N@sA{59LbfI40<5fv;NHolIjJ~*reJ#n58j)HNq{@7lvkEI2GFFtT1qo8Pr2Fvy9 z-nIooo_-*&Y^ozl&t>^%GN`y^)0hbn-k{Oq-+BR<(reCbw6rL7F)K`i|8H{c09vWA zC1qfcDo5YAj~QmN*Sy5D#!E9V-3L&%b1&pr_C0oo#0N0zRw5A?;UmpQoiRpO29B zIINt8@`-{zL_f|FEb=u*u91?E08;ni70Juza0CC?qtOX?F|Cx^2f2KnyOD+QyN_2L1 zzVq^(y1FmvZ7`pMmox<0_SX_78$NPqJ1<24xCG5m7Tzn!%F-@?GSv+4D)KdlO2Ah4 zW%<988u;&|BDo^uLE+7-UcCGF=Xk+#{*vEvr98qBKf1=~yjIxyhj-HYu=;RHwnHZN z(R@81^!DEL93LOwwgvBo#R>RTfO=-q&NI@E0#xy{T;HaqH|5$Pq}X6y4s`{X{Mbo{ zdQF2g`sCyU5J!S;5x&$OaZwYW8t={bd<LSg&)zjJIRQoUl}5av;>&fI;&pw1th;o073kJ1j%|O$TyRJ54;I}1{n?X_J;j30|3P>7clid zYKc3{1yv2X3$iZC|7Gyv*$Su;6%~bVdI}2(ZS`X&#(DGKvQtiQS#p7e0`1YhzVlMO z35Ku)AZW}s@f6#tyWz1q4Y=2hR@cj>Hym?7f8*PPZer90_+SVE<|_^GtEgYcfhzzV zPkCqgv7`pwOclW`H${vI^y^eOR`fg}I^G34i!iXc=0ChXKjs!K zvvYItq%O78)V}?j%o5K3Oa`OU(rQ2{`udcr9KblUH7bj{$Lf0Syo|>uBehfGeR~auxT%r!*nL#;;z! ztOt32>6!nBwq+5tD$<4mb_eX!+fTGvKe4vS%tm8uQ!4qDE-LZ=p8U`u&G+`_4rt~v z^LQKBOGDz9;19MZ8M%W(LpMQ}g>4ibFhs|oH4h5CJ33@1GznTAH%wkW7WdH38Jcar z8js}v8BZ-|$cZx`AmI9aQ#c&X?m|#G8KMrfypFHl1PLmNd3kuk*v4({OEYN~ujXcD z-I=%qT&=o$H1nUgt~Rg|?v*C7(m!LQ<$nP(1Ahr^q@-|kF{chq@!-e^&W0ywjQlAY z{OlEX3$w;`B*^2aa7qv%1;6tXxsuWYWp`@ljzlhN*Vt>Z~osvv;Uqk z{_x+UcK!_ny&B%j>dH7_5jg)F==as|b$rbRg=9AX@5{e;>NP0Q1eW&4_525z8U3@1 z;Jg3)yM6EKd6=IF8UeqIZ@ZD6W-+U29!0QfT4^?#1?Ja_Q~pb>BthEQ(`MqqFaOSRW{R=jbqH-9uv@WBx0;*Mdf4)&q zTr|(jCNerYpoj~bF@r#h420r~iXPZ&<#@Ps=yRpGV-(W;WEi01?A1G_NAj#9_2{YeS5w<5-Vbi zb^a$!$zSYj*;-m(m$S&UB-h=Tt!zRE3|?wsV|g9bt=op*clf>d!T)$eVd5N^^j}t2 z>;CJ4aPwF<$jR_2$P~y{$?#|bO|VCh1Xfkyooib9-euEdy{|GzI>keNYn<;H@*5^Hz36{2x;yk zw9_wqTv30m*a$4!_}wcFd0E&y?MrBw>#OwajqEKjjRrG5T<(EB&*rnDyUSVlX@u(- z4L<C=`LV1T`HfY1-A?!V0qw&)$v^onMxWj7ZV-5Kdfc*^T&^)5M?>~$hSZ5?!3eI zeGLq-W#*iNkPo>TxKB#8b-dk1ft6ZX`y}{>grwK1S>*$-c@-v>U`d2|EX`XPMa9cD zDKXpjyGA!VqgMA%NOhfNEc8ytFb3*a3Xq^}x}k-mqoY0U8|9=mXWty>i$LTY~lH-4W1gZfikV6%QTE__FCWA67X!J#wj;6VNezw#pc{M2&j zyj8i3&R5iHPbOeJq5v*~!d-U6xJRJg%|zO*nOL2pQS%!**)s`7kn&FYTTPc5muAgk zf`#dV)MMwS1zT)qE`W8%#0KJ?8B{d`f!`I@#^0Sx#6%_lXOUAYSlK`H7`=JH(?2-a z6!aeSAR1jboPgbDJX?c^Ybhz|gIZfYLR+X9GN1>KHaX=@~%(8V^!31IyB& zC)UxjqG!kZDbUF;jDdUPLn!7QtxXARIDxh{;KzDr=aaariTD6ck|`)y*3vte%|#IW z5V&E6>JIw$_PpE<>#M7VvE%CpaO(>HDDmU@$l_msEILaY^zIfsG7J@MITI3S(@-z! ze-QLw&OAg_i+Rcb4C3>M4aeD;@g(=R=G&AXw>JeaI(QSKgML%j07~dg-(d!v^snyL zUY>8B4tp;DjBQJ=J*})wqXPvg*BQ%azEQd{j|>fQ8qNVGtd9zk2-vN(Ca1eY<(mJq zX8m8>Fd?Sd3CpUv|5b|$%xk*1+BHUBq1=wZ&3K~*16fH5I$y*CgAOc;p3;b zc0g7Gl#8ifdF%ho-n7X^Q-%)^dBOnCG3W$^l1p*{4sVwUj6(`IGA0g!p!W^OzK(-1 zm?}xcnaI_;W2Op>9%eY@$nB6DSZ9~(ldhIpkn`5<(qGYLQF(asb-gcc+{)PV?{-uc=3vU!l zD-ZCSmjzraf)gih0VeS^+w%9NgDG;`j#PLHdiYJ3Sj4V*coj}F90)@2`pw7l0Zvqz$a;FefO3(2BYqMU+lTH zJD@q4#mS_F(t{`7&A$bn?$7NHtmQ3thFBo8rObp8uwJIN`J`i@>pEOiovWO*}YE!R536ql7|nURdoVfKArVB;3>=q@;n%YG$(*4} z7&mN3K`@m(Z0ElF591P)$ywg&?tN5}8QJoqmA|*Mq2g%G;7q!mPeT=_wbHF?-G zt?s!?g66n$?B3BaTF6=6ujCK~UJHvg{NGB<0ALU)GXRK_dk&p~daq;oDNw0gz6fZ` zXGehhJ-52%G4OvLpFoFd`0L-%M7w)m;?J3Tm{Ak2Cwpi-aI`%J_d~jW0UiDj@@j}!MW3vfPt99I*AT- zpo+zEh-D`k#5Ed2Tx$_lB6K??|AP=J3oC!}te5B2t~2fnGa~%?TDZ8WvT;4u>KBrd z6sxK$1;|vl%$2VnZrQRv0|wdDAEQ?TSh%$(Lj^|ntL#^YA)pu&RQ8<#819o14ADAx z=kUfV-A&LFlP~W_Mh2u=v`+w2KyK}-d7Q&l_p z16qG#sGF#$snuIR!)6zi2X@i_MEZiBbi!%u8ER@u_qg$smAo5c5K9;HEHwoln|r!e zR_#Im1E4hqx87FjkPthrE*1n|Nan?H%qA3(9!4W5ML1mz>GcIcHdW>*W|$~@Zq{me#LVU%9a zv+;A?tap;wPmgHq^0u*r`CK(3b+%4k+TeXLU(pNEo=-aWFgtct3arb!G>uug)*Ovi zS@6TR@A9(~$&suUInUiVk>cHW_ouav9zd%j^Fn1e0N8UGYF`+*FMG$6EU%|c>q*F zXGh1G$E6VePh-pq%U`XqnX--!>xvCHT|tfJNm;t_EX&=WXG257LVAoL4?Fc3Y#cf( zO;tP-i38C|v@^dZI6<$WDW1yph~dXGR09Z!*i2QGFUp%CFss&! zk;aRJ=l;lFHq$kLv~u54zOzZAT%q8(XKAAk%yDuST{jqI8#^>K-I-{SQ@blMu&zrr z3IjXDn}Nx2>Su20MS$b&ZTKUzOg*=y@iL&|SRd|VY*flplZLsOnZ|fAih==US4Cl0 z1-3{SeV;$_y^0Fa?VH1;yum~eb*)w9{bdan6r>m zK`tu1vCxMMRCCDjn8#zKr6=$$#KwFuCDRb*B=MRDI?ItA6BBce3*uTalqwn;jIG)g zc4pMB{9&w!{Q@r8F@*m# zyzS%AcFB<5%PLPl#*mqxup8!jHcx>j(cA$q9kk+kc{-hrjp0CVxR!d~88n1mh!b@^ zU@1!kCR`$9d1Xb@x(oC`WWRL=VD6k9fsq_gk|HEhT{sVU;LrWYrb1X3Eb8iyZkg#9 zG@n4{>Kyd~;K7kCM9ahp;LgeicAIFeV9V+RhCC+Vr)?vTiPrsYm}XbP?Tv5lo|kwyFWu&@HI<@;_zU(e zV1;&dPJN|0nJiD?1Ju5g04W`BIrG`^fmFxLn?-2bCCMG6Nxqr(>(}+tt!Rl>4@-EJ zH9e@DNrXd`G#x>%+g+`i$$E@GckUBpuK@z*n9l^*3&2?oKOPYtmt zw2N`z!}$YUFZGQAia2dVwsp-7zCJZKPDGUTNejyE-NrtIVN6OH=iqG4IJJK)NFwoy z%(vu1avUfM)N=#i{^m(&P#kvgB<;%A9c3Kxf)9wNuWu}1TLDh9+RP_W&f?pRqUo8L zPJr#~tVhX&i@O+=XS0D4ARU*c5b4M}yJWBgO^RMYJC~$5F%h{EuZnX-lT(z;7>waQ zO=OQ|mrwQFqw)lQxgaZJUce4v^;P`z$BoNNeq-Psn3c5As736i@CSC&l%)6|75}=# zM~UN~PqXt2*&!N2`MJDj{1v_idpG$%Q5v?0L6&}ab?F#QX7J@>YHDf&`0d2(k}*gJ zzvkjs|I2@|tBSo{a~~Fa+Wq_cMZo)XdCBJq5b1!3$7Ks-H=Iz(Bb2+;U+aB1~fvD$hKZ1V%o}IcO$t9}?%CZ?^em zU-7lASvV(O&`m?hlvtCPN=wn#>guomp%hCFKlrIF;Fxq*`fIoP0AUnqGw7S@Y7Y;O z3mmB>T6~hTv$Kfrs2Cw;bi&q;p~a}BRn9+Klzp`{p6jtG{ZTmnhGV=8A+AI9!BcfHF<}`}lHRerg zCfF6s;UtF@ki8$WC(8QGj;-m;4?gZcH$%?CFSOMm0E+_wgU_4(IG*f?<`1>h%HsZu1_wO3eJ7ZctMRL@>ROoqwcAq?^BmoI>rm2X#-^7A|3_-Oc$0RZ^nWR-yc zXzf`*cwH8WM&c0;FY0I@m~i{c5U&(+{HBV#9>&6LWQP_2L{?W;#&iYQ8sq^0Zu8<| ziQ;6a-Ez7)bDkPtvm&O5K}7KFizSKHVJpybkMDA0h}&(c3yql6hxJ52x|L*6Q|l~3 zQY#YwYgfJP*fV6UP10=BPYs4P6>}d>Hzfk(YJ4-Vl<_<=~>|zEW z@D~D~aVouK@dZ+nbgJuQ=R~uQX~#hPX5T zDP$Qah5aCjfltotX0rM@w##J!9Dhd9sJCNaEScOK*XdduVs~TmvzR}-Z=sS#|D}W> zyfl}A7f1)nb?9fnWobUuf6A`Bw-V5JON*gcwFsIU-*m%PGk6wIsq1FL`%nCru4u>fcJU6ed)F=wh6TV_TPNqC{c@P6Pf;(s< zzPc1QH#f5*qx!2_XZgvZC*YbvyBkI=LR6@gD8QxlFApoz??arv zHzE9q8lm#)_v4N^!s^}%%-|$^Z&_yHnjEF=EyfTLfO|X<@>)IaH!K&H0oP25S_KaV z8V!4Ty1TP0?`xl;+@ElLtET41ck`n2zQKjIu&^`pJ%Z}Sr(Q$chPogF+vu!t;DDH> z`uu3dgAmOdQ(t1Qnp;^VO*#%eM+%#bm(_cNJyoHD^QF-GbnA&rD%^T&5$8qudl({% z#iF_QO1pgc*ZM!S{SHMh_sn^qmEXP(pkJ1Kx1c86JXK|=-Y709I2j1ResAc`I>ksb z63&oBz>F&dgRc0yBN|bV8{e+KbBxKrN@;3pCdeioLSE4u;()y;5Ni!0eOT{L!=>r& zBAuSO_6PnS2N=8NI(7FEztTiH^^ z866cEPjo0o<>W0SkUPo`S|Gk|5B}Aola1-mY~1514km(7+IDWvIzzFtvLeG{_dqsS zE5tY3gO$V(>q!s%2F2-Ffb^Ye5_^H!w*|Ye6&`R(Bh-4*FSlrJOH8&{E}!+)lAWeq zAYT3R<+DSt<_@}O>tt8K>Pcr1ttkP=+6m2S5P$G=&f`bLLWXJ5(3B?)~reE(20`F!yq4NNPfS3EFUQ$@mZ zVWJAYITk*9TOGH5w1Wj7)fNLH(Ux+A*2YFcsdes`hpoS9kpo)zg>d%tIws(8^WAqW zw(7V-zbr2sEJHl&S~5K7QR__3U%~ZtZ>{kEOTMe~)A&@yfT$2SWC_zh%2{k5x4(}q z@3^eBVcK)bq59p==K&Y5j<;Sh{>1Vu+)0aZgNP5aW5`N>fUH(eopnS~8=n!4WPf}k zdIX@l;T>?NyI$KGk*HCqLp}IM-n=ynRKYfBC+(?M3&~3E?gH2W9qtNSRuui-50xQFzg}k>IUpQ$7}2irMt*e4GFMA?8X;8@llIxu=v<>Ph4J$hnf{s+FU1ME;w@H9+z2D)lqhn_e+>1w)bj7M%3qr}bGwa`f zVF4)zVK@%?a;5hn>M6!=lWj2@nz9iZ-mf9>)Boof-q?>TCgv>dW|n(z7`19fvFb>Y zSwf@Ufr;x@v;0VAB+Uv06m`fKiO|8kRo8hX;T{}TmfG;~oM}mq!cfu)B%5Q6zv^q5 zzjZ^Gg>e?8L|CmO`Z-&h$e-QQ>;GxwIaWS`w!nw@M`egi7-fvqB_I(I>dYCCYrn?w za&Jaoy8>eDx^i#_U2)n}xAlz&t;H{pRb%wEB+_7AHmp`W9>+v=oSAr?nxu z8T4{;CWLr$J){$`##`7b{)FJ;SEM9e4}N5oDoPF`o$XD}j8H?blM z;yPZ^)0wYzPFGai7+lIuv1&?Lyw4pQd&lf;a_n~GFE1koha{N%&k3YKU0n5#4{0q$Phxkq}dWEVv~oV#tJUYW~%C&KAH zh#047bN*2#@zHGb2(vUt`T>)Nr}kFqs0KgJ9Z-`D7QRJuaH;}rQ&``?Xyr4$@`Pt8 zHg@JQ^^hi>J;>-;u}VowdewaeD_z0~)hRuzL95H!F+oLUC-G_{yXhG8&u+eZyCt}& z5Emh`TZ8EOBD>w#_hhLAzDCQm4PtF^-fMI@(Xba6RIpg>L_r7VEP>2!W81@=smwst zk%F1{X6d?53pLBk4|z+Y2tO{85CY!p|4l$xfq=^nw=8cgASN)J`@R5;lezo9$a$WmikHcT`F&tz z#jP!II8mwLYHrj@(XeQ`bHb(F4aHYY;WV!Y-X7J*ZAsHQXhXNIxy6fGa*=)a#5qBl zaO7dLF)ylN%j)u?jDmKwUkJ^}d&sp1HP?2|`ech@hl6axz*BH7{)OyxqOQd8(2Irj z7RIf}EF=4|na%`ZsB(t|0pZVY-#Ve<73^%uog3k4%0*%abMmt-vQ!6d=H1Q^L`4F5 z>rmKlRu%iKw8tla`lB4}B*AP8XUo6|SC!w)c>afr(*~RLgdOLtusVlOqy6AzJcVz1 zM!x*#o7?Tf`^ru#$)e9wfkYD9=jKgho-AK?!f%T0H=xOCV*)KNK&B&HQSYlrH$oDf zvYc9#UjHVUv>oIb`Q)y%3oNI4i;$C!l-02Lq)vzm6sr@EoZs`?y891;k3+o};q-Iy z0N*8)LI@oDZ0mgqAI86!MSyu8Mlz9k?|S{?Y@<#|tV0QI;<@vZzBh#2z7vcqZp$}s z;2$6Gue#~B?T@4R_afTY+l__aJucLSvj3}ti|eC&hJ0V2nouKPyCMm6-6n4r9^s7d z)ZG`02ohYR?H3Ww{iIz=xU(a6am(Wn{VisRpC>TplqBB z<8~;y*Z7>O=~>ynZTi>y|Flv_k=vsVjEl}r)fD8rPPBB0-Rcpfp5skGGDTo}N)j|4*NW$89%#CQY)$!LM xgzmMGjbwP&hAuCn8CJ(p*Z-dyh0+b@@Hi>h\n", - "#include \n", - "#include " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "std::frexp(x, &n) => 16.4 = 0.5125 * (1 << 5)" - ] - } - ], - "source": [ - "#include \n", - "#include \n", - "\n", - "double x, y;\n", - "int n;\n", - "x = 16.4;\n", - "y = frexp(x, &n);\n", - "std::cout << \"std::frexp(x, &n) => \" \n", - " << x << \" = \" << y << \" * \"\n", - " << \"(1 << \" << n << \")\";" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "16.400000" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - " 0.5125 * (1 << 5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "std::pair GetFixedPointMultiplierShift(double double_multiplier) {\n", - " int32_t significand, exponent;\n", - " if (double_multiplier == 0.) {\n", - " significand = 0;\n", - " exponent = 0;\n", - " return std::make_pair(significand, exponent);\n", - " }\n", - "\n", - " // Get the significand and exponent.\n", - " double significand_d = std::frexp(double_multiplier, &exponent);\n", - "\n", - " // Convert the double significand to int significand, i.e., convert into a\n", - " // integer where the decimal point is between bit 31 and 30. This is done by\n", - " // multiplying the double value with 2^31 and then casting to int.\n", - " significand_d = std::round(significand_d * (1ll << 31));\n", - " auto significand_int64 = static_cast(significand_d);\n", - " ICHECK_LE(significand_int64, (1ll << 31));\n", - " if (significand_int64 == (1ll << 31)) {\n", - " significand_int64 /= 2;\n", - " ++exponent;\n", - " }\n", - " ICHECK_LE(significand_int64, std::numeric_limits::max());\n", - " significand = static_cast(significand_int64);\n", - " return std::make_pair(significand, exponent);\n", - "}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "C++14", - "language": "C++14", - "name": "xcpp14" - }, - "language_info": { - "codemirror_mode": "text/x-c++src", - "file_extension": ".cpp", - "mimetype": "text/x-c++src", - "name": "c++", - "version": "14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/doc/tutorials/index.md b/doc/tutorials/index.md index c618c7de..fd6d29d7 100644 --- a/doc/tutorials/index.md +++ b/doc/tutorials/index.md @@ -9,7 +9,7 @@ frontend/index pass/index roofline/index deploy/index -auto-quantize/index +aq/index pattern/index partition/index vta/index diff --git a/doc/tutorials/intro/custom-op.ipynb b/doc/tutorials/intro/custom-op.ipynb index 51dfdae5..956fa5dd 100644 --- a/doc/tutorials/intro/custom-op.ipynb +++ b/doc/tutorials/intro/custom-op.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -84,11 +84,11 @@ " %conv.weight : Float(8, 3, 1, 1, strides=[3, 1, 1, 1], requires_grad=1, device=cpu)):\n", " %/conv/Conv_output_0 : Float(1, 8, 8, 8, strides=[512, 64, 8, 1], requires_grad=0, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1], onnx_name=\"/conv/Conv\"](%data, %conv.weight), scope: __main__.M::/torch.nn.modules.conv.Conv2d::conv # /media/pc/data/tmp/cache/conda/envs/py312x/lib/python3.12/site-packages/torch/nn/modules/conv.py:456:0\n", " %/pool/GlobalAveragePool_output_0 : Float(1, 8, 1, 1, strides=[8, 1, 1, 1], requires_grad=1, device=cpu) = onnx::GlobalAveragePool[onnx_name=\"/pool/GlobalAveragePool\"](%/conv/Conv_output_0), scope: __main__.M::/torch.nn.modules.pooling.AdaptiveAvgPool2d::pool # /media/pc/data/tmp/cache/conda/envs/py312x/lib/python3.12/site-packages/torch/nn/functional.py:1260:0\n", - " %/Constant_output_0 : Long(4, strides=[1], requires_grad=0, device=cpu) = onnx::Constant[value= 1 1 1 8 [ CPULongType{4} ], onnx_name=\"/Constant\"](), scope: __main__.M:: # /tmp/ipykernel_3479623/1160623730.py:16:0\n", - " %/Reshape_output_0 : Float(1, 1, 1, 8, strides=[8, 8, 8, 1], requires_grad=1, device=cpu) = onnx::Reshape[allowzero=0, onnx_name=\"/Reshape\"](%/pool/GlobalAveragePool_output_0, %/Constant_output_0), scope: __main__.M:: # /tmp/ipykernel_3479623/1160623730.py:16:0\n", + " %/Constant_output_0 : Long(4, strides=[1], requires_grad=0, device=cpu) = onnx::Constant[value= 1 1 1 8 [ CPULongType{4} ], onnx_name=\"/Constant\"](), scope: __main__.M:: # /tmp/ipykernel_3941401/1160623730.py:16:0\n", + " %/Reshape_output_0 : Float(1, 1, 1, 8, strides=[8, 8, 8, 1], requires_grad=1, device=cpu) = onnx::Reshape[allowzero=0, onnx_name=\"/Reshape\"](%/pool/GlobalAveragePool_output_0, %/Constant_output_0), scope: __main__.M:: # /tmp/ipykernel_3941401/1160623730.py:16:0\n", " %/Softmax_output_0 : Float(1, 1, 1, 8, strides=[8, 8, 8, 1], requires_grad=1, device=cpu) = onnx::Softmax[axis=3, onnx_name=\"/Softmax\"](%/Reshape_output_0), scope: __main__.M:: # /media/pc/data/tmp/cache/conda/envs/py312x/lib/python3.12/site-packages/torch/nn/functional.py:1885:0\n", - " %/Constant_1_output_0 : Long(2, strides=[1], requires_grad=0, device=cpu) = onnx::Constant[value= 1 8 [ CPULongType{2} ], onnx_name=\"/Constant_1\"](), scope: __main__.M:: # /tmp/ipykernel_3479623/1160623730.py:18:0\n", - " %output : Float(1, 8, strides=[8, 1], requires_grad=1, device=cpu) = onnx::Reshape[allowzero=0, onnx_name=\"/Reshape_1\"](%/Softmax_output_0, %/Constant_1_output_0), scope: __main__.M:: # /tmp/ipykernel_3479623/1160623730.py:18:0\n", + " %/Constant_1_output_0 : Long(2, strides=[1], requires_grad=0, device=cpu) = onnx::Constant[value= 1 8 [ CPULongType{2} ], onnx_name=\"/Constant_1\"](), scope: __main__.M:: # /tmp/ipykernel_3941401/1160623730.py:18:0\n", + " %output : Float(1, 8, strides=[8, 1], requires_grad=1, device=cpu) = onnx::Reshape[allowzero=0, onnx_name=\"/Reshape_1\"](%/Softmax_output_0, %/Constant_1_output_0), scope: __main__.M:: # /tmp/ipykernel_3941401/1160623730.py:18:0\n", " return (%output)\n", "\n" ] @@ -650,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -673,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -702,7 +702,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -733,7 +733,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -879,7 +879,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -893,7 +893,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -907,16 +907,16 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "GenericFunc(0x94baf50)" + "GenericFunc(0x92187f0)" ] }, - "execution_count": 24, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -934,7 +934,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -973,17 +973,17 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[0.117226 , 0.13859256, 0.1350793 , 0.12436351, 0.12440445,\n", - " 0.1263574 , 0.13254553, 0.10143131]], dtype=float32)" + "array([[0.13445556, 0.12515777, 0.11348397, 0.12063695, 0.13582838,\n", + " 0.12997034, 0.11783069, 0.12263636]], dtype=float32)" ] }, - "execution_count": 26, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -994,17 +994,17 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[0.117226 , 0.13859256, 0.13507928, 0.1243635 , 0.12440444,\n", - " 0.1263574 , 0.13254553, 0.10143131]], dtype=float32)" + "array([[0.13445556, 0.12515777, 0.11348397, 0.12063695, 0.13582838,\n", + " 0.12997034, 0.1178307 , 0.12263636]], dtype=float32)" ] }, - "execution_count": 27, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1015,17 +1015,17 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[0.117226 , 0.13859256, 0.13507928, 0.1243635 , 0.12440444,\n", - " 0.1263574 , 0.13254553, 0.10143131]], dtype=float32)" + "array([[0.13445556, 0.12515777, 0.11348397, 0.12063695, 0.13582838,\n", + " 0.12997034, 0.1178307 , 0.12263636]], dtype=float32)" ] }, - "execution_count": 28, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1043,7 +1043,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -1056,7 +1056,7 @@ " %3 = cast(%2, dtype="int8") /* ty=Tensor[(1, 3, 8, 8), int8] */;\n", " %4 = nn.conv2d(%3, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), int8] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1], out_dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n", " %5 = cast(%4, dtype="int64") /* ty=Tensor[(1, 8, 8, 8), int64] */;\n", - " %6 = fixed_point_multiply(%5, multiplier=1157502080, shift=-7) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n", + " %6 = fixed_point_multiply(%5, multiplier=1192595968, shift=-7) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n", " %7 = clip(%6, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n", " %8 = cast(%7, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n", " %9 = cast(%8, dtype="int8") /* ty=Tensor[(1, 8, 8, 8), int8] */;\n", diff --git a/doc/tutorials/intro/custom-quant-op.ipynb b/doc/tutorials/intro/custom-quant-op.ipynb deleted file mode 100644 index 166bfba5..00000000 --- a/doc/tutorials/intro/custom-quant-op.ipynb +++ /dev/null @@ -1,842 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 自定义 relay 量化算子(python)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from testing import viz_expr # 可视化 relay" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from tvm.relay.testing import run_infer_type\n", - "from tvm.relay.dataflow_pattern import (\n", - " wildcard, is_op,\n", - " # FunctionPattern,\n", - " DFPatternCallback,\n", - " rewrite\n", - ")\n", - "import tvm\n", - "from tvm.ir.attrs import DictAttrs\n", - "from tvm import relay, te, topi\n", - "from tvm.relay.op import op as _op\n", - "from tvm.target import generic_func\n", - "\n", - "@generic_func\n", - "def schedule_special_op(attrs, outs, target):\n", - " with target:\n", - " outs = [outs] if isinstance(outs, te.tensor.Tensor) else outs\n", - " output = outs[0]\n", - " sch = te.create_schedule(output.op) \n", - " return sch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```{topic} 主题\n", - "尽可能仅仅使用 Python 实现 Relay 算子的定义。\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from d2py.utils.file import mkdir\n", - "root_dir = \".temp\"\n", - "mkdir(f\"{root_dir}/logs\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exported graph: graph(%data : Float(1, 3, 8, 8, strides=[192, 64, 8, 1], requires_grad=0, device=cpu),\n", - " %conv.weight : Float(8, 3, 1, 1, strides=[3, 1, 1, 1], requires_grad=1, device=cpu)):\n", - " %/conv/Conv_output_0 : Float(1, 8, 8, 8, strides=[512, 64, 8, 1], requires_grad=0, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1], onnx_name=\"/conv/Conv\"](%data, %conv.weight), scope: __main__.M::/torch.nn.modules.conv.Conv2d::conv # /media/pc/data/tmp/cache/conda/envs/py312x/lib/python3.12/site-packages/torch/nn/modules/conv.py:456:0\n", - " %/pool/GlobalAveragePool_output_0 : Float(1, 8, 1, 1, strides=[8, 1, 1, 1], requires_grad=1, device=cpu) = onnx::GlobalAveragePool[onnx_name=\"/pool/GlobalAveragePool\"](%/conv/Conv_output_0), scope: __main__.M::/torch.nn.modules.pooling.AdaptiveAvgPool2d::pool # /media/pc/data/tmp/cache/conda/envs/py312x/lib/python3.12/site-packages/torch/nn/functional.py:1260:0\n", - " %/Constant_output_0 : Long(4, strides=[1], requires_grad=0, device=cpu) = onnx::Constant[value= 1 1 1 8 [ CPULongType{4} ], onnx_name=\"/Constant\"](), scope: __main__.M:: # /tmp/ipykernel_4084674/1160623730.py:16:0\n", - " %/Reshape_output_0 : Float(1, 1, 1, 8, strides=[8, 8, 8, 1], requires_grad=1, device=cpu) = onnx::Reshape[allowzero=0, onnx_name=\"/Reshape\"](%/pool/GlobalAveragePool_output_0, %/Constant_output_0), scope: __main__.M:: # /tmp/ipykernel_4084674/1160623730.py:16:0\n", - " %/Softmax_output_0 : Float(1, 1, 1, 8, strides=[8, 8, 8, 1], requires_grad=1, device=cpu) = onnx::Softmax[axis=3, onnx_name=\"/Softmax\"](%/Reshape_output_0), scope: __main__.M:: # /media/pc/data/tmp/cache/conda/envs/py312x/lib/python3.12/site-packages/torch/nn/functional.py:1885:0\n", - " %/Constant_1_output_0 : Long(2, strides=[1], requires_grad=0, device=cpu) = onnx::Constant[value= 1 8 [ CPULongType{2} ], onnx_name=\"/Constant_1\"](), scope: __main__.M:: # /tmp/ipykernel_4084674/1160623730.py:18:0\n", - " %output : Float(1, 8, strides=[8, 1], requires_grad=1, device=cpu) = onnx::Reshape[allowzero=0, onnx_name=\"/Reshape_1\"](%/Softmax_output_0, %/Constant_1_output_0), scope: __main__.M:: # /tmp/ipykernel_4084674/1160623730.py:18:0\n", - " return (%output)\n", - "\n" - ] - } - ], - "source": [ - "import torch\n", - "from torch.nn import functional as F\n", - "from torch import nn\n", - "from torch.onnx import OperatorExportTypes, utils\n", - "\n", - "class M(nn.Module):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.conv = nn.Conv2d(3, 8, 1, 1, 0, bias=False, groups=1)\n", - " self.pool = nn.AdaptiveAvgPool2d((1, 1))\n", - "\n", - " def forward(self, x):\n", - " x = self.conv(x)\n", - " x = self.pool(x)\n", - " b, c, h, w = x.shape\n", - " x = x.view((b, h, w, c))\n", - " x = F.softmax(x, dim=3)\n", - " x = x.view((b, h * w * c))\n", - " return x\n", - "\n", - "model = M()\n", - "model.eval()\n", - "\n", - "shape = 1, 3, 8, 8\n", - "input_name = \"data\"\n", - "xx = torch.rand(*shape, dtype=torch.float32, requires_grad=False)\n", - "# model = torch.jit.trace(model, xx)\n", - "# 导出模型\n", - "output_name = \"test\"\n", - "utils.export(\n", - " model, # torch 模型\n", - " xx, # 模型输入或者对于多个输入，使用元组\n", - " f\"{root_dir}/{output_name}.onnx\", # 模型保存的位置（可以是文件或类似文件的对象）\n", - " export_params=True, # 将训练后的参数权重存储在模型文件内\n", - " opset_version=17, # 导出模型的 ONNX 版本\n", - " do_constant_folding=True, # 是否执行常量折叠以进行优化\n", - " input_names = [input_name], # 模型的输入名称\n", - " output_names = ['output'], # 模型的输出名称\n", - " keep_initializers_as_inputs=True,\n", - " # export_modules_as_functions=True,\n", - " verbose=True,\n", - " operator_export_type=OperatorExportTypes.ONNX_FALLTHROUGH,\n", - " # dynamic_axes={'data' : {0 : 'batch_size'}, # 可变长度的轴\n", - " # 'output' : {0 : 'batch_size'}}\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "前端导入：" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "

def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = nn.global_avg_pool2d(%0) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=/Reshape:0:0 */;\n",
-       "  %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=/Softmax:0:0 */;\n",
-       "  reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=/Reshape_1:0:0 */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = nn.global_avg_pool2d(%0) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %2 = nn.softmax(%1, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %3 = transpose(%2, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%3, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import numpy as np\n", - "import onnx\n", - "import tvm\n", - "from tvm import relay\n", - "\n", - "class Reshape4dSoftmaxReshape2dRewrite(DFPatternCallback):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.x = wildcard()\n", - " self.reshape4d = is_op(\"reshape\")(self.x) # 将 NCHW 转换为 NHWC，其他 H=W=1\n", - " self.softmax = is_op(\"nn.softmax\")(self.reshape4d)\n", - " self.softmax_axis = self.softmax.has_attr({\"axis\": 3})\n", - " self.reshape2d = is_op(\"reshape\")(self.softmax_axis)\n", - " self.pattern = self.reshape2d\n", - "\n", - " def callback(self, pre, post, node_map):\n", - " x = node_map[self.x][0]\n", - " relay.transform.InferTypeLocal(x).shape\n", - " x = relay.nn.softmax(x, axis=1)\n", - " relay.transform.InferTypeLocal(x)\n", - " x = relay.transpose(x, (0, 2, 3, 1))\n", - " relay.transform.InferTypeLocal(x)\n", - " x = relay.reshape(x, (1, -1))\n", - " relay.transform.InferTypeLocal(x)\n", - " return x\n", - " \n", - "onnx_model = onnx.load(f\"{root_dir}/{output_name}.onnx\")\n", - "mod, params = relay.frontend.from_onnx(onnx_model, {input_name: shape}, freeze_params=True)\n", - "mod = relay.transform.InferType()(mod)\n", - "mod.show()\n", - "mod[\"main\"] = rewrite(Reshape4dSoftmaxReshape2dRewrite(), mod[\"main\"])\n", - "mod.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 未成功量化 ``softmax_transpose_reshape2d`` 结构" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:autotvm:One or more operators have not been tuned. Please tune your model for better performance. Use DEBUG logging level to see more details.\n" - ] - }, - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = multiply(%data, 36.7488f /* ty=float32 */) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
-       "  %1 = round(%0) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
-       "  %2 = clip(%1, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
-       "  %3 = cast(%2, dtype="int8") /* ty=Tensor[(1, 3, 8, 8), int8] */;\n",
-       "  %4 = nn.conv2d(%3, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), int8] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1], out_dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
-       "  %5 = cast(%4, dtype="int64") /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
-       "  %6 = fixed_point_multiply(%5, multiplier=1188167424, shift=-6) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
-       "  %7 = clip(%6, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
-       "  %8 = cast(%7, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
-       "  %9 = cast(%8, dtype="int8") /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
-       "  %10 = annotation.stop_fusion(%9) /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
-       "  %11 = cast(%10, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
-       "  %12 = nn.global_avg_pool2d(%11) /* ty=Tensor[(1, 8, 1, 1), int32] */;\n",
-       "  %13 = cast(%12, dtype="float32") /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %14 = multiply(%13, 0.0128598f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %15 = nn.softmax(%14, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %16 = transpose(%15, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%16, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from dataclasses import dataclass\n", - "\n", - "@dataclass\n", - "class Dataset:\n", - " input_name: str\n", - " shape: tuple\n", - "\n", - " def __iter__(self):\n", - " for _ in range(2):\n", - " yield {self.input_name: np.random.normal(0, 1, size=self.shape).astype(\"float32\")}\n", - "\n", - "dataset = Dataset(input_name, shape)\n", - "\n", - "with tvm.transform.PassContext(opt_level=3):\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ):\n", - " qmod = relay.quantize.quantize(mod, params, dataset)\n", - "qmod.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 为 `nn.softmax` 添加分区规则" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "原始的 `nn.global_avg_pool2d` 规则把其后所有算子均视为非量化算子：" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = annotation.stop_fusion(%0) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
-       "  %2 = nn.global_avg_pool2d(%1) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %3 = nn.softmax(%2, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %4 = transpose(%3, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%4, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "quant_passes = tvm.transform.Sequential([relay.quantize.partition(), relay.quantize.annotate()])\n", - "with tvm.transform.PassContext(\n", - " opt_level=3, \n", - " required_pass=[\"QuantizeAnnotate\", \"QuantizeCalibrate\", \"QuantizeRealize\"]):\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " annotate_mod = quant_passes(mod)\n", - "annotate_mod.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "此时无法对 `nn.softmax` 进行量化，需要重置 `nn.global_avg_pool2d` 分区规则。" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from tvm.relay.quantize._partition import (\n", - " register_partition_function, \n", - " partition_expr_check,\n", - " QPartitionExpr\n", - ")\n", - "from tvm.relay.quantize.quantize import _forward_op\n", - "\n", - "relay.op.get(\"nn.global_avg_pool2d\").reset_attr(\"FQPartitionRewrite\")\n", - "def pool2d_partition_function(ref_call, new_args, ctx):\n", - " cond, expr = partition_expr_check(new_args[0])\n", - " if cond:\n", - " expr = new_args[0].realize()\n", - " return QPartitionExpr(_forward_op(ref_call, [expr]))\n", - " return None\n", - "\n", - "register_partition_function(\"nn.global_avg_pool2d\", pool2d_partition_function)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = annotation.stop_fusion(%0) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
-       "  %2 = nn.global_avg_pool2d(%1) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %3 = annotation.stop_fusion(%2) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %4 = nn.softmax(%3, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %5 = transpose(%4, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%5, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "quant_passes = tvm.transform.Sequential([relay.quantize.partition(), relay.quantize.annotate()])\n", - "with tvm.transform.PassContext(\n", - " opt_level=3, \n", - " required_pass=[\"QuantizeAnnotate\", \"QuantizeCalibrate\", \"QuantizeRealize\"]):\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " annotate_mod = quant_passes(mod)\n", - "annotate_mod.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "@register_partition_function(\"nn.softmax\")\n", - "def softmax_partition_function(ref_call, new_args, ctx):\n", - " \"\"\"Rewrite function for softmax for partition\"\"\"\n", - " data_cond, data = partition_expr_check(new_args[0])\n", - "\n", - " if data_cond:\n", - " data = new_args[0].realize()\n", - " ret = _forward_op(ref_call, [data])\n", - " return QPartitionExpr(ret)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = annotation.stop_fusion(%0) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
-       "  %2 = nn.global_avg_pool2d(%1) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %3 = annotation.stop_fusion(%2) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %4 = nn.softmax(%3, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %6 = transpose(%5, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%6, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "quant_passes = tvm.transform.Sequential([relay.quantize.partition(), relay.quantize.annotate()])\n", - "with tvm.transform.PassContext(\n", - " opt_level=3, \n", - " required_pass=[\"QuantizeAnnotate\", \"QuantizeCalibrate\", \"QuantizeRealize\"]):\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " annotate_mod = quant_passes(mod)\n", - "annotate_mod.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 为 `nn.softmax` 添加注解规则" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "重置 `nn.global_avg_pool2d` 注解规则" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = annotation.stop_fusion(%0) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
-       "  %2 = nn.global_avg_pool2d(%1) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %3 = annotation.stop_fusion(%2) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %4 = nn.softmax(%3, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %6 = transpose(%5, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%6, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from tvm.relay.quantize._calibrate import calibrate\n", - "\n", - "dataset = Dataset(input_name, shape)\n", - "\n", - "with tvm.transform.PassContext(\n", - " opt_level=3, \n", - " required_pass=[\"QuantizeAnnotate\"]):\n", - " \n", - " quant_passes = tvm.transform.Sequential([\n", - " relay.quantize.partition(), relay.quantize.annotate(), \n", - " ])\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " annotate_mod = quant_passes(mod)\n", - "annotate_mod.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "._register..frewrite_with_guard(ref_call, new_args, ctx)>" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from tvm.relay.quantize._annotate import (\n", - " register_annotate_function,\n", - " _get_expr_kind, QAnnotateExpr,\n", - "\n", - ")\n", - "from tvm.relay.quantize.quantize import quantize_context, QAnnotateKind\n", - "\n", - "def annotate_identity_rewrite(ref_call, new_args, ctx):\n", - " \"\"\"Rewrite function for avg_pool2d\"\"\"\n", - " if quantize_context().check_to_skip(ref_call):\n", - " return None\n", - "\n", - " expr, x_kind = _get_expr_kind(new_args[0])\n", - " if x_kind is None:\n", - " return None\n", - " expr = _forward_op(ref_call, [expr])\n", - " return QAnnotateExpr(expr, QAnnotateKind.ACTIVATION)\n", - "\n", - "relay.op.get(\"nn.global_avg_pool2d\").reset_attr(\"FQAnnotateRewrite\")\n", - "register_annotate_function(\"nn.global_avg_pool2d\", annotate_identity_rewrite)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = annotation.stop_fusion(%0) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
-       "  %2 = nn.global_avg_pool2d(%1) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %3 = annotation.stop_fusion(%2) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %4 = nn.softmax(%3, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %6 = transpose(%5, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%6, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from tvm.relay.quantize._calibrate import calibrate\n", - "\n", - "# dataset = Dataset(input_name, shape)\n", - "\n", - "with tvm.transform.PassContext(\n", - " opt_level=3, \n", - " required_pass=[\"QuantizeAnnotate\"]):\n", - " \n", - " quant_passes = tvm.transform.Sequential([\n", - " relay.quantize.partition(), relay.quantize.annotate(), \n", - " ])\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " annotate_mod = quant_passes(mod)\n", - "annotate_mod.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "from tvm.relay.quantize._annotate import attach_simulated_quantize\n", - "\n", - "@register_annotate_function(\"nn.softmax\")\n", - "def softmax_rewrite(ref_call, new_args, ctx):\n", - " \"\"\"Rewrite function for softmax. Lhs of nn.softmax will be quantized to\n", - " input field.\n", - " Output would be in activation field\"\"\"\n", - " if quantize_context().check_to_skip(ref_call):\n", - " return None\n", - "\n", - " lhs_expr, lhs_kind = _get_expr_kind(new_args[0])\n", - "\n", - " if lhs_kind is None or lhs_kind == QAnnotateKind.ACTIVATION:\n", - " lhs_expr = attach_simulated_quantize(lhs_expr, QAnnotateKind.INPUT)\n", - "\n", - " expr = _forward_op(ref_call, [lhs_expr])\n", - "\n", - " return QAnnotateExpr(expr, QAnnotateKind.ACTIVATION)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */;\n",
-       "  %1 = annotation.stop_fusion(%0) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
-       "  %2 = nn.global_avg_pool2d(%1) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */;\n",
-       "  %3 = annotation.stop_fusion(%2) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %4 = nn.softmax(%3, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %6 = transpose(%5, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
-       "  reshape(%6, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from tvm.relay.quantize._calibrate import calibrate\n", - "\n", - "# dataset = Dataset(input_name, shape)\n", - "\n", - "with tvm.transform.PassContext(\n", - " opt_level=3, \n", - " required_pass=[\"QuantizeAnnotate\"]):\n", - " \n", - " quant_passes = tvm.transform.Sequential([\n", - " relay.quantize.partition(), relay.quantize.annotate(), \n", - " ])\n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " annotate_mod = quant_passes(mod)\n", - "annotate_mod.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] {\n",
-       "  %0 = multiply(%data, 55.8021f /* ty=float32 */) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
-       "  %1 = round(%0) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
-       "  %2 = clip(%1, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
-       "  %3 = cast(%2, dtype="int8") /* ty=Tensor[(1, 3, 8, 8), int8] */;\n",
-       "  %4 = nn.conv2d(%3, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), int8] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1], out_dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
-       "  %5 = cast(%4, dtype="int64") /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
-       "  %6 = fixed_point_multiply(%5, multiplier=1647147392, shift=-7) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
-       "  %7 = clip(%6, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
-       "  %8 = cast(%7, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
-       "  %9 = cast(%8, dtype="int8") /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
-       "  %10 = annotation.stop_fusion(%9) /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
-       "  %11 = cast(%10, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
-       "  %12 = nn.global_avg_pool2d(%11) /* ty=Tensor[(1, 8, 1, 1), int32] */;\n",
-       "  %13 = cast(%12, dtype="int64") /* ty=Tensor[(1, 8, 1, 1), int64] */;\n",
-       "  %14 = fixed_point_multiply(%13, multiplier=1101593600, shift=4) /* ty=Tensor[(1, 8, 1, 1), int64] */;\n",
-       "  %15 = clip(%14, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 1, 1), int64] */;\n",
-       "  %16 = cast(%15, dtype="int32") /* ty=Tensor[(1, 8, 1, 1), int32] */;\n",
-       "  %17 = cast(%16, dtype="int8") /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
-       "  %18 = annotation.stop_fusion(%17) /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
-       "  %19 = cast(%18, dtype="float32") /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %20 = multiply(%19, 0.00148864f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %21 = nn.softmax(%20, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %22 = multiply(%21, 937.98f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %23 = round(%22) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %24 = clip(%23, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
-       "  %25 = cast(%24, dtype="int8") /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
-       "  %26 = annotation.stop_fusion(%25) /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
-       "  %27 = transpose(%26, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), int8] */;\n",
-       "  %28 = reshape(%27, newshape=[1, -1]) /* ty=Tensor[(1, 8), int8] */;\n",
-       "  %29 = cast(%28, dtype="float32") /* ty=Tensor[(1, 8), float32] */;\n",
-       "  multiply(%29, 0.00106612f /* ty=float32 */) /* ty=Tensor[(1, 8), float32] */\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from tvm.relay.quantize._calibrate import calibrate\n", - "\n", - "dataset = Dataset(input_name, shape)\n", - "\n", - "with tvm.transform.PassContext(opt_level=3):\n", - " \n", - " with relay.quantize.qconfig(\n", - " skip_conv_layers=[],\n", - " calibrate_mode=\"kl_divergence\", \n", - " weight_scale=\"max\",\n", - " # round_for_shift=True,\n", - " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", - " skip_dense_layer=False,\n", - " ): \n", - " qmod = relay.quantize.quantize(mod, params=params, dataset=dataset)\n", - "qmod.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "py312x", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/doc/tutorials/intro/index.md b/doc/tutorials/intro/index.md index 20ec5270..4c5cff8f 100644 --- a/doc/tutorials/intro/index.md +++ b/doc/tutorials/intro/index.md @@ -5,7 +5,7 @@ env packed-func trace custom-op -custom-quant-op +rewrite-quant-op custom-vta-op Conv2dTransposeReshapeConcat hard-swish-v1 diff --git a/doc/tutorials/intro/rewrite-quant-op.ipynb b/doc/tutorials/intro/rewrite-quant-op.ipynb new file mode 100644 index 00000000..f037eae9 --- /dev/null +++ b/doc/tutorials/intro/rewrite-quant-op.ipynb @@ -0,0 +1,1060 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 重写 relay 量化算子(python)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from testing import viz_expr # 可视化 relay" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import tvm\n", + "@tvm.instrument.pass_instrument\n", + "class PrintIR:\n", + " \"\"\"仅在传递执行之前，打印传递名称、IR。\"\"\"\n", + " def run_before_pass(self, mod, info):\n", + " print(f\"Running pass: {info}->{mod['main']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from tvm.relay.testing import run_infer_type\n", + "from tvm.relay.dataflow_pattern import (\n", + " wildcard, is_op, is_constant,\n", + " # FunctionPattern,\n", + " DFPatternCallback,\n", + " rewrite\n", + ")\n", + "import tvm\n", + "from tvm.ir.attrs import DictAttrs\n", + "from tvm import relay, te, topi\n", + "from tvm.relay.op import op as _op\n", + "from tvm.target import generic_func\n", + "\n", + "@generic_func\n", + "def schedule_special_op(attrs, outs, target):\n", + " with target:\n", + " outs = [outs] if isinstance(outs, te.tensor.Tensor) else outs\n", + " output = outs[0]\n", + " sch = te.create_schedule(output.op) \n", + " return sch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```{topic} 主题\n", + "尽可能仅仅使用 Python 实现 Relay 算子的定义。\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from d2py.utils.file import mkdir\n", + "root_dir = \".temp\"\n", + "mkdir(f\"{root_dir}/logs\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running pass: The meta data of the pass - pass name: sequential, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* span=aten___convolution_0_data:0:0 */) {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0], padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* span=aten__view_1:0:0 */\n", + "}\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* span=aten___convolution_0_data:0:0 */) {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0], padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* span=aten__view_1:0:0 */\n", + "}\n", + "\n", + "Running pass: The meta data of the pass - pass name: SimplifyInference, opt_level: 0, required passes: [\n", + "InferType, ]\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: FoldConstant, opt_level: 2, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: FoldScaleAxis, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: BackwardFoldScaleAxis, opt_level: 3, required passes: [\n", + "InferType, ]\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: ForwardFoldScaleAxis, opt_level: 3, required passes: [\n", + "InferType, ]\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: FoldConstant, opt_level: 2, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: CanonicalizeOps, opt_level: 3, required passes: [\n", + "InferType, ]\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: FoldConstant, opt_level: 2, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "Running pass: The meta data of the pass - pass name: InferType, opt_level: 0, required passes: []\n", + "->fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import torch\n", + "from torch.nn import functional as F\n", + "from torch import nn\n", + "\n", + "\n", + "class M(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.conv = nn.Conv2d(3, 8, 1, 1, 0, bias=False, groups=1)\n", + " self.pool = nn.AdaptiveAvgPool2d((1, 1))\n", + "\n", + " def forward(self, x):\n", + " x = self.conv(x)\n", + " x = self.pool(x)\n", + " b, c, h, w = x.shape\n", + " x = x.view((b, h, w, c))\n", + " x = F.softmax(x, dim=3)\n", + " x = x.view((b, h * w * c))\n", + " return x\n", + "\n", + "model = M()\n", + "model.eval()\n", + "\n", + "shape = 1, 3, 8, 8\n", + "input_name = \"data\"\n", + "dtype = \"float32\"\n", + "data_np = np.random.rand(*shape).astype(dtype)\n", + "with torch.no_grad():\n", + " pt_model = M().eval().float()\n", + " traced_model = torch.jit.trace(pt_model, torch.from_numpy(data_np)).eval()\n", + "mod, params = relay.frontend.from_pytorch(traced_model, [(\"data\", shape)], \n", + " use_parser_friendly_name=True)\n", + "with tvm.transform.PassContext(opt_level=3, instruments=[PrintIR()]):\n", + " mod = relay.quantize.prerequisite_optimize(mod, params)\n", + "print(mod['main'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "前端导入：" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__view_0:0:0 */;\n", + " %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=aten__softmax_0:0:0 */;\n", + " reshape(%3, newshape=[1, 8]) /* ty=Tensor[(1, 8), float32] span=aten__view_1:0:0 */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n", + "fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n", + " %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %1 = nn.adaptive_avg_pool2d(%0, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %2 = nn.softmax(%1, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %3 = transpose(%2, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n", + " reshape(%3, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32]) -> Tensor[(1, 8), float32] */\n", + "\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import tvm\n", + "from tvm import relay\n", + "\n", + "class Reshape4dSoftmaxReshape2dRewrite(DFPatternCallback):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.x = wildcard()\n", + " self.reshape4d = is_op(\"reshape\")(self.x) # 将 NCHW 转换为 NHWC，其他 H=W=1\n", + " self.softmax = is_op(\"nn.softmax\")(self.reshape4d)\n", + " self.softmax_axis = self.softmax.has_attr({\"axis\": 3})\n", + " self.reshape2d = is_op(\"reshape\")(self.softmax_axis)\n", + " self.pattern = self.reshape2d\n", + "\n", + " def callback(self, pre, post, node_map):\n", + " x = node_map[self.x][0]\n", + " relay.transform.InferTypeLocal(x).shape\n", + " x = relay.nn.softmax(x, axis=1)\n", + " relay.transform.InferTypeLocal(x)\n", + " x = relay.transpose(x, (0, 2, 3, 1))\n", + " relay.transform.InferTypeLocal(x)\n", + " x = relay.reshape(x, (1, -1))\n", + " relay.transform.InferTypeLocal(x)\n", + " return x\n", + "\n", + "mod = relay.transform.InferType()(mod)\n", + "print(mod['main'])\n", + "mod[\"main\"] = rewrite(Reshape4dSoftmaxReshape2dRewrite(), mod[\"main\"])\n", + "print(mod['main'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 未成功量化 ``softmax_transpose_reshape2d`` 结构" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:autotvm:One or more operators have not been tuned. Please tune your model for better performance. Use DEBUG logging level to see more details.\n" + ] + }, + { + "data": { + "text/html": [ + "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n",
+       "  %0 = multiply(%data, 41.4911f /* ty=float32 */) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %1 = round(%0) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %2 = clip(%1, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %3 = cast(%2, dtype="int8") /* ty=Tensor[(1, 3, 8, 8), int8] */;\n",
+       "  %4 = nn.conv2d(%3, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), int8] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1], out_dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
+       "  %5 = cast(%4, dtype="int64") /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %6 = fixed_point_multiply(%5, multiplier=1960450048, shift=-7) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %7 = clip(%6, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %8 = cast(%7, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
+       "  %9 = cast(%8, dtype="int8") /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
+       "  %10 = annotation.stop_fusion(%9) /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
+       "  %11 = cast(%10, dtype="float32") /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
+       "  %12 = multiply(%11, 0.0148817f /* ty=float32 */) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
+       "  %13 = nn.adaptive_avg_pool2d(%12, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n",
+       "  %14 = nn.softmax(%13, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %15 = transpose(%14, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n",
+       "  reshape(%15, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from dataclasses import dataclass\n", + "\n", + "@dataclass\n", + "class Dataset:\n", + " input_name: str\n", + " shape: tuple\n", + "\n", + " def __iter__(self):\n", + " for _ in range(2):\n", + " yield {self.input_name: np.random.normal(0, 1, size=self.shape).astype(\"float32\")}\n", + "\n", + "dataset = Dataset(input_name, shape)\n", + "\n", + "with tvm.transform.PassContext(opt_level=3):\n", + " with relay.quantize.qconfig(\n", + " skip_conv_layers=[],\n", + " calibrate_mode=\"kl_divergence\", \n", + " weight_scale=\"max\",\n", + " # round_for_shift=True,\n", + " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", + " skip_dense_layer=False,\n", + " ):\n", + " qmod = relay.quantize.quantize(mod, params, dataset)\n", + "qmod.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 为 `nn.softmax` 添加分区规则" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "原始的 `nn.global_avg_pool2d` 规则把其后所有算子均视为非量化算子：" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */, %dom_scale: float32 /* ty=float32 */, %clip_min: float32 /* ty=float32 */, %clip_max: float32 /* ty=float32 */, %dom_scale1: float32 /* ty=float32 */, %clip_min1: float32 /* ty=float32 */, %clip_max1: float32 /* ty=float32 */, %dom_scale2: float32 /* ty=float32 */, %clip_min2: float32 /* ty=float32 */, %clip_max2: float32 /* ty=float32 */) -> Tensor[(1, 8), float32] {\n", + " %0 = relay.op.annotation.simulated_quantize(%data, %dom_scale, %clip_min, %clip_max, kind=1) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n", + " %1 = relay.op.annotation.simulated_quantize(meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, %dom_scale1, %clip_min1, %clip_max1, kind=2) /* ty=Tensor[(8, 3, 1, 1), float32] */;\n", + " %2 = nn.conv2d(%0, %1, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %3 = relay.op.annotation.simulated_quantize(%2, %dom_scale2, %clip_min2, %clip_max2, kind=1) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %4 = annotation.cast_hint(%3, dtype=\"int8\") /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %6 = nn.adaptive_avg_pool2d(%5, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %7 = nn.softmax(%6, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %8 = transpose(%7, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n", + " reshape(%8, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32], float32, float32, float32, float32, float32, float32, float32, float32, float32) -> Tensor[(1, 8), float32] */\n", + "\n" + ] + } + ], + "source": [ + "quant_passes = tvm.transform.Sequential([relay.quantize.partition(), relay.quantize.annotate()])\n", + "with tvm.transform.PassContext(opt_level=3):\n", + " with relay.quantize.qconfig(\n", + " skip_conv_layers=[],\n", + " calibrate_mode=\"kl_divergence\", \n", + " weight_scale=\"max\",\n", + " # round_for_shift=True,\n", + " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", + " skip_dense_layer=False,\n", + " ): \n", + " # run_mod = relay.quantize.prerequisite_optimize(mod, params)\n", + " annotate_mod = quant_passes(mod)\n", + "print(annotate_mod[\"main\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from tvm.relay.quantize._partition import (\n", + " register_partition_function, \n", + " partition_expr_check,\n", + " QPartitionExpr\n", + ")\n", + "from tvm.relay.quantize.quantize import _forward_op" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def avg_pool2d_partition_function(ref_call, new_args, ctx):\n", + " cond, expr = partition_expr_check(new_args[0])\n", + " if cond:\n", + " expr = new_args[0].realize()\n", + " return QPartitionExpr(_forward_op(ref_call, [expr]))\n", + " return None\n", + "\n", + "# register_partition_function(\"nn.avg_pool2d\", avg_pool2d_partition_function)\n", + "_op.get(\"nn.adaptive_avg_pool2d\").reset_attr(\"FQPartitionRewrite\")\n", + "register_partition_function(\"nn.adaptive_avg_pool2d\", avg_pool2d_partition_function)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "@register_partition_function(\"nn.softmax\")\n", + "def softmax_partition_function(ref_call, new_args, ctx):\n", + " \"\"\"Rewrite function for softmax for partition\"\"\"\n", + " data_cond, data = partition_expr_check(new_args[0])\n", + "\n", + " if data_cond:\n", + " data = new_args[0].realize()\n", + " ret = _forward_op(ref_call, [data])\n", + " return QPartitionExpr(ret)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */, %dom_scale: float32 /* ty=float32 */, %clip_min: float32 /* ty=float32 */, %clip_max: float32 /* ty=float32 */, %dom_scale1: float32 /* ty=float32 */, %clip_min1: float32 /* ty=float32 */, %clip_max1: float32 /* ty=float32 */, %dom_scale2: float32 /* ty=float32 */, %clip_min2: float32 /* ty=float32 */, %clip_max2: float32 /* ty=float32 */) -> Tensor[(1, 8), float32] {\n", + " %0 = relay.op.annotation.simulated_quantize(%data, %dom_scale, %clip_min, %clip_max, kind=1) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n", + " %1 = relay.op.annotation.simulated_quantize(meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, %dom_scale1, %clip_min1, %clip_max1, kind=2) /* ty=Tensor[(8, 3, 1, 1), float32] */;\n", + " %2 = nn.conv2d(%0, %1, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %3 = relay.op.annotation.simulated_quantize(%2, %dom_scale2, %clip_min2, %clip_max2, kind=1) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %4 = annotation.cast_hint(%3, dtype=\"int8\") /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %6 = nn.adaptive_avg_pool2d(%5, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %7 = annotation.stop_fusion(%6) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %8 = nn.softmax(%7, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %9 = annotation.stop_fusion(%8) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %10 = transpose(%9, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n", + " reshape(%10, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32], float32, float32, float32, float32, float32, float32, float32, float32, float32) -> Tensor[(1, 8), float32] */\n", + "\n" + ] + } + ], + "source": [ + "quant_passes = tvm.transform.Sequential([relay.quantize.partition(), relay.quantize.annotate()])\n", + "with tvm.transform.PassContext(\n", + " opt_level=3, \n", + " required_pass=[\"QuantizeAnnotate\", \"QuantizeCalibrate\", \"QuantizeRealize\"]):\n", + " with relay.quantize.qconfig(\n", + " skip_conv_layers=[],\n", + " calibrate_mode=\"kl_divergence\", \n", + " weight_scale=\"max\",\n", + " # round_for_shift=True,\n", + " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", + " skip_dense_layer=False,\n", + " ): \n", + " annotate_mod = quant_passes(mod)\n", + "print(annotate_mod['main'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 为 `nn.softmax` 添加注解规则" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from tvm.relay.quantize._annotate import (\n", + " attach_simulated_quantize, register_annotate_function, \n", + " QAnnotateKind, _get_expr_kind, QAnnotateExpr\n", + ")\n", + "from tvm.relay.quantize.quantize import quantize_context\n", + "\n", + "def avg_pool2d_rewrite(ref_call, new_args, ctx):\n", + " \"\"\"Rewrite function for max pool2d\"\"\"\n", + " if quantize_context().check_to_skip(ref_call):\n", + " return None\n", + "\n", + " expr, x_kind = _get_expr_kind(new_args[0])\n", + " if x_kind is None:\n", + " return None\n", + " expr = _forward_op(ref_call, [expr])\n", + " return QAnnotateExpr(expr, QAnnotateKind.ACTIVATION)\n", + "\n", + "# register_annotate_function(\"nn.avg_pool2d\", avg_pool2d_rewrite)\n", + "_op.get(\"nn.adaptive_avg_pool2d\").reset_attr(\"FQAnnotateRewrite\")\n", + "register_annotate_function(\"nn.adaptive_avg_pool2d\", avg_pool2d_rewrite)\n", + "\n", + "@register_annotate_function(\"nn.softmax\")\n", + "def softmax_rewrite(ref_call, new_args, ctx):\n", + " \"\"\"Rewrite function for softmax. Lhs of nn.softmax will be quantized to\n", + " input field.\n", + " Output would be in activation field\"\"\"\n", + " if quantize_context().check_to_skip(ref_call):\n", + " return None\n", + "\n", + " lhs_expr, lhs_kind = _get_expr_kind(new_args[0])\n", + "\n", + " if lhs_kind is None or lhs_kind == QAnnotateKind.ACTIVATION:\n", + " lhs_expr = attach_simulated_quantize(lhs_expr, QAnnotateKind.INPUT)\n", + "\n", + " expr = _forward_op(ref_call, [lhs_expr])\n", + "\n", + " return QAnnotateExpr(expr, QAnnotateKind.ACTIVATION)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fn (%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */, %dom_scale: float32 /* ty=float32 */, %clip_min: float32 /* ty=float32 */, %clip_max: float32 /* ty=float32 */, %dom_scale1: float32 /* ty=float32 */, %clip_min1: float32 /* ty=float32 */, %clip_max1: float32 /* ty=float32 */, %dom_scale2: float32 /* ty=float32 */, %clip_min2: float32 /* ty=float32 */, %clip_max2: float32 /* ty=float32 */, %dom_scale3: float32 /* ty=float32 */, %clip_min3: float32 /* ty=float32 */, %clip_max3: float32 /* ty=float32 */, %dom_scale4: float32 /* ty=float32 */, %clip_min4: float32 /* ty=float32 */, %clip_max4: float32 /* ty=float32 */) -> Tensor[(1, 8), float32] {\n", + " %0 = relay.op.annotation.simulated_quantize(%data, %dom_scale, %clip_min, %clip_max, kind=1) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n", + " %1 = relay.op.annotation.simulated_quantize(meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] */, %dom_scale1, %clip_min1, %clip_max1, kind=2) /* ty=Tensor[(8, 3, 1, 1), float32] */;\n", + " %2 = nn.conv2d(%0, %1, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=aten___convolution_0:0:0 */;\n", + " %3 = relay.op.annotation.simulated_quantize(%2, %dom_scale2, %clip_min2, %clip_max2, kind=1) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %4 = annotation.cast_hint(%3, dtype=\"int8\") /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %5 = annotation.stop_fusion(%4) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n", + " %6 = nn.adaptive_avg_pool2d(%5, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n", + " %7 = relay.op.annotation.simulated_quantize(%6, %dom_scale3, %clip_min3, %clip_max3, kind=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %8 = annotation.cast_hint(%7, dtype=\"int8\") /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %9 = annotation.stop_fusion(%8) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %10 = nn.softmax(%9, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %11 = relay.op.annotation.simulated_quantize(%10, %dom_scale4, %clip_min4, %clip_max4, kind=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %12 = annotation.cast_hint(%11, dtype=\"int8\") /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %13 = annotation.stop_fusion(%12) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n", + " %14 = transpose(%13, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), float32] */;\n", + " reshape(%14, newshape=[1, -1]) /* ty=Tensor[(1, 8), float32] */\n", + "} /* ty=fn (Tensor[(1, 3, 8, 8), float32], float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32, float32) -> Tensor[(1, 8), float32] */\n", + "\n" + ] + } + ], + "source": [ + "# from tvm.relay.quantize._calibrate import calibrate\n", + "\n", + "# dataset = Dataset(input_name, shape)\n", + "\n", + "with tvm.transform.PassContext(\n", + " opt_level=3, \n", + " required_pass=[\"QuantizeAnnotate\"]):\n", + " \n", + " quant_passes = tvm.transform.Sequential([\n", + " relay.quantize.partition(), relay.quantize.annotate(), \n", + " ])\n", + " with relay.quantize.qconfig(\n", + " skip_conv_layers=[],\n", + " calibrate_mode=\"kl_divergence\", \n", + " weight_scale=\"max\",\n", + " # round_for_shift=True,\n", + " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", + " skip_dense_layer=False,\n", + " ): \n", + " annotate_mod = quant_passes(mod)\n", + "print(annotate_mod['main'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 校验量化 softmax 结果" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n",
+       "  %0 = multiply(%data, 49.2086f /* ty=float32 */) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %1 = round(%0) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %2 = clip(%1, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %3 = cast(%2, dtype="int8") /* ty=Tensor[(1, 3, 8, 8), int8] */;\n",
+       "  %4 = nn.conv2d(%3, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), int8] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1], out_dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
+       "  %5 = cast(%4, dtype="int64") /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %6 = fixed_point_multiply(%5, multiplier=1640946048, shift=-7) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %7 = clip(%6, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %8 = cast(%7, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
+       "  %9 = cast(%8, dtype="int8") /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
+       "  %10 = annotation.stop_fusion(%9) /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
+       "  %11 = cast(%10, dtype="float32") /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
+       "  %12 = multiply(%11, 0.014991f /* ty=float32 */) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
+       "  %13 = nn.adaptive_avg_pool2d(%12, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n",
+       "  %14 = multiply(%13, 840.53f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %15 = round(%14) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %16 = clip(%15, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %17 = cast(%16, dtype="int8") /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
+       "  %18 = annotation.stop_fusion(%17) /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
+       "  %19 = cast(%18, dtype="float32") /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %20 = multiply(%19, 0.00118973f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %21 = nn.softmax(%20, axis=1) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %22 = multiply(%21, 934.276f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %23 = round(%22) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %24 = clip(%23, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %25 = cast(%24, dtype="int8") /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
+       "  %26 = annotation.stop_fusion(%25) /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
+       "  %27 = transpose(%26, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), int8] */;\n",
+       "  %28 = reshape(%27, newshape=[1, -1]) /* ty=Tensor[(1, 8), int8] */;\n",
+       "  %29 = cast(%28, dtype="float32") /* ty=Tensor[(1, 8), float32] */;\n",
+       "  multiply(%29, 0.00107035f /* ty=float32 */) /* ty=Tensor[(1, 8), float32] */\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# from tvm.relay.quantize._calibrate import calibrate\n", + "\n", + "dataset = Dataset(input_name, shape)\n", + "\n", + "with tvm.transform.PassContext(opt_level=3):\n", + " with relay.quantize.qconfig(\n", + " skip_conv_layers=[],\n", + " calibrate_mode=\"kl_divergence\", \n", + " weight_scale=\"max\",\n", + " # round_for_shift=True,\n", + " # rounding=\"TONEAREST\", # \"UPWARD\" or \"TONEAREST\"\n", + " skip_dense_layer=False,\n", + " ): \n", + " qmod = relay.quantize.quantize(mod, params=params, dataset=dataset)\n", + "qmod.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "data = np.random.normal(0, 1, size=shape).astype(\"float32\")\n", + "torch_out = model(torch.from_numpy(data)).detach().numpy()\n", + "\n", + "\n", + "target = 'llvm'\n", + "dev = tvm.device(target, 0)\n", + "\n", + "# 原始模型\n", + "with tvm.transform.PassContext(opt_level=3):\n", + " lib = relay.build(mod, target, params=params)\n", + "func = lib[lib.libmod_name]\n", + "module = tvm.contrib.graph_executor.GraphModule(func(dev))\n", + "module.run(**{input_name: data})\n", + "float_output = module.get_output(0).numpy()\n", + "\n", + "# 量化的模型\n", + "with tvm.transform.PassContext(opt_level=3):\n", + " lib = relay.build(qmod, target, params=params)\n", + "func = lib[lib.libmod_name]\n", + "module = tvm.contrib.graph_executor.GraphModule(func(dev))\n", + "module.run(**{input_name: data})\n", + "quant_output = module.get_output(0).numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.128034 , 0.12699334, 0.11690528, 0.12342793, 0.12102022,\n", + " 0.11912127, 0.13309336, 0.13140461]], dtype=float32)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch_out" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.12589025, 0.13976204, 0.12616956, 0.12493419, 0.11583884,\n", + " 0.11355935, 0.1261162 , 0.12772952]], dtype=float32)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "float_output" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.12630105, 0.13593417, 0.12630105, 0.1252307 , 0.11559757,\n", + " 0.11452722, 0.12630105, 0.1273714 ]], dtype=float32)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "quant_output" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "class QSoftmaxRewrite(DFPatternCallback):\n", + " \"\"\"变换 dequantize+softmax+quantize` 为 `qnn.softmax`\"\"\"\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.x = wildcard()\n", + " self.cast = is_op(\"cast\")(self.x).has_attr({\"dtype\": \"float32\"})\n", + " self.scale = is_constant()\n", + " self.multiply = is_op(\"multiply\")(self.cast, self.scale)\n", + " self.softmax = is_op(\"nn.softmax\")(self.multiply)\n", + " self.output_scale = is_constant()\n", + " self.multiply_out = is_op(\"multiply\")(self.softmax, self.output_scale)\n", + " self.round = is_op(\"round\")(self.multiply_out)\n", + " self.clip = is_op(\"clip\")(self.round)\n", + " self.cast_out = is_op(\"cast\")(self.clip).has_attr({\"dtype\": \"int8\"})\n", + " self.stop_fusion = is_op(\"annotation.stop_fusion\")(self.cast_out)\n", + " self.pattern = self.stop_fusion\n", + "\n", + " def callback(self, pre, post, node_map):\n", + " x = node_map[self.x][0]\n", + " softmax = node_map[self.softmax][0]\n", + " scale = node_map[self.scale][0]\n", + " output_scale = node_map[self.output_scale][0]\n", + " output_scale = output_scale.data\n", + " output_scale = relay.const(1.0/output_scale.numpy(), dtype=output_scale.dtype)\n", + " zero_point = relay.const(0, dtype=\"int32\")\n", + " output_zero_point = relay.const(0, dtype=\"int32\")\n", + " out = relay.qnn.softmax(x, scale, zero_point, output_scale, output_zero_point, axis=softmax.attrs.axis)\n", + " return out" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=aten___convolution_0_data:0:0 */) -> Tensor[(1, 8), float32] {\n",
+       "  %0 = multiply(%data, 49.2086f /* ty=float32 */) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %1 = round(%0) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %2 = clip(%1, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 3, 8, 8), float32] */;\n",
+       "  %3 = cast(%2, dtype="int8") /* ty=Tensor[(1, 3, 8, 8), int8] */;\n",
+       "  %4 = nn.conv2d(%3, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), int8] */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1], out_dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
+       "  %5 = cast(%4, dtype="int64") /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %6 = fixed_point_multiply(%5, multiplier=1640946048, shift=-7) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %7 = clip(%6, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 8, 8), int64] */;\n",
+       "  %8 = cast(%7, dtype="int32") /* ty=Tensor[(1, 8, 8, 8), int32] */;\n",
+       "  %9 = cast(%8, dtype="int8") /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
+       "  %10 = annotation.stop_fusion(%9) /* ty=Tensor[(1, 8, 8, 8), int8] */;\n",
+       "  %11 = cast(%10, dtype="float32") /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
+       "  %12 = multiply(%11, 0.014991f /* ty=float32 */) /* ty=Tensor[(1, 8, 8, 8), float32] */;\n",
+       "  %13 = nn.adaptive_avg_pool2d(%12, output_size=[1, 1]) /* ty=Tensor[(1, 8, 1, 1), float32] span=aten__adaptive_avg_pool2d_0:0:0 */;\n",
+       "  %14 = multiply(%13, 840.53f /* ty=float32 */) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %15 = round(%14) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %16 = clip(%15, a_min=-127f, a_max=127f) /* ty=Tensor[(1, 8, 1, 1), float32] */;\n",
+       "  %17 = cast(%16, dtype="int8") /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
+       "  %18 = annotation.stop_fusion(%17) /* ty=Tensor[(1, 8, 1, 1), int8] */;\n",
+       "  %19 = qnn.softmax(%18, 0.00118973f /* ty=float32 */, 0, 0.00107035f, 0, axis=1);\n",
+       "  %20 = transpose(%19, axes=[0, 2, 3, 1]) /* ty=Tensor[(1, 1, 1, 8), int8] */;\n",
+       "  %21 = reshape(%20, newshape=[1, -1]) /* ty=Tensor[(1, 8), int8] */;\n",
+       "  %22 = cast(%21, dtype="float32") /* ty=Tensor[(1, 8), float32] */;\n",
+       "  multiply(%22, 0.00107035f /* ty=float32 */) /* ty=Tensor[(1, 8), float32] */\n",
+       "}\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "qmod[\"main\"] = rewrite(QSoftmaxRewrite(), qmod[\"main\"])\n", + "qmod.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "# QNN 量化的模型\n", + "with tvm.transform.PassContext(opt_level=3):\n", + " lib = relay.build(qmod, target, params=params)\n", + "func = lib[lib.libmod_name]\n", + "module = tvm.contrib.graph_executor.GraphModule(func(dev))\n", + "module.run(**{input_name: data})\n", + "qnn_output = module.get_output(0).numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.12589025, 0.12630105, 0.1252307 ],\n", + " [0.13976204, 0.13593417, 0.13272314],\n", + " [0.12616956, 0.12630105, 0.1252307 ],\n", + " [0.12493419, 0.1252307 , 0.1252307 ],\n", + " [0.11583884, 0.11559757, 0.11666792],\n", + " [0.11355935, 0.11452722, 0.10917548],\n", + " [0.1261162 , 0.12630105, 0.1252307 ],\n", + " [0.12772952, 0.1273714 , 0.1252307 ]], dtype=float32)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.concatenate([float_output, quant_output, qnn_output]).T" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py312x", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/tvm_book/special/rewriter/softmax.py b/src/tvm_book/special/rewriter/softmax.py index 3af1ac43..5517e664 100644 --- a/src/tvm_book/special/rewriter/softmax.py +++ b/src/tvm_book/special/rewriter/softmax.py @@ -10,25 +10,6 @@ class Reshape4dSoftmaxReshape2dRewrite(DFPatternCallback): """简化 `reshape4d_softmax_reshape2d` 为 `softmax_reshape` - - 原始mod: - ``` - def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] { - %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */; - %1 = nn.global_avg_pool2d(%0) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */; - %2 = reshape(%1, newshape=[1, 1, 1, 8]) /* ty=Tensor[(1, 1, 1, 8), float32] span=/Reshape:0:0 */; - %3 = nn.softmax(%2, axis=3) /* ty=Tensor[(1, 1, 1, 8), float32] span=/Softmax:0:0 */; - reshape(%3, newshape=[-1, 8]) /* ty=Tensor[(1, 8), float32] span=/Reshape_1:0:0 */ - } - ``` - 简化后为： - ``` - def @main(%data: Tensor[(1, 3, 8, 8), float32] /* ty=Tensor[(1, 3, 8, 8), float32] span=/conv/Conv.data:0:0 */) -> Tensor[(1, 8), float32] { - %0 = nn.conv2d(%data, meta[relay.Constant][0] /* ty=Tensor[(8, 3, 1, 1), float32] span=/conv/Conv.conv.weight:0:0 */, padding=[0, 0, 0, 0], channels=8, kernel_size=[1, 1]) /* ty=Tensor[(1, 8, 8, 8), float32] span=/conv/Conv:0:0 */; - %1 = nn.global_avg_pool2d(%0) /* ty=Tensor[(1, 8, 1, 1), float32] span=/pool/GlobalAveragePool:0:0 */; - softmax_reshape(%1, __dict__={"axis"=1, "newshape"=[1, 8]}) /* ty=Tensor[(1, 8), float32] */ - } - ``` """ def __init__(self): super().__init__()