From f75489eb413bc4a5acce373d0bd1c1c794aa4ce0 Mon Sep 17 00:00:00 2001 From: frayle-ons <194791647+frayle-ons@users.noreply.github.com> Date: Wed, 24 Jun 2026 10:42:41 +0100 Subject: [PATCH 1/8] added evaluation demo notebook, demo query data, and updated demo readme --- DEMO/README.md | 19 ++ DEMO/data/fake_soc_eval_queries.csv | 49 ++++ DEMO/evaluation_workflow_demo.ipynb | 382 ++++++++++++++++++++++++++++ 3 files changed, 450 insertions(+) create mode 100644 DEMO/data/fake_soc_eval_queries.csv create mode 100644 DEMO/evaluation_workflow_demo.ipynb diff --git a/DEMO/README.md b/DEMO/README.md index b3b0f8f..948069e 100644 --- a/DEMO/README.md +++ b/DEMO/README.md @@ -66,6 +66,25 @@ It covers: * A showcase of the pre-made `RagHook` class and performing different RAG type tasks on `VectorStoreSearchOutput` results including classification, reranking and keyword identication, and how to customise the specific task using this class. +### 5. Evaluating VectorStore Performance with Metrics : `evaluation_workflow_demo.ipynb` + +This notebook demonstrates how to use the Evaluation module to assess the performance of one or more VectorStore instances against ground-truth labelled data. + +It covers: + +* An introduction to the Evaluation module and its multi-class, single-label classification focus. + +* The available evaluation metrics + +* Creating multiple VectorStore instances with varying data coverage to showcase performance differences. + +* Instantiating an `Evaluation` object with ground truth data and selected metrics. + +* Running the `evaluate()` method to compute metrics across multiple VectorStores. + +* Memory-efficient evaluation using callable functions to load VectorStores on-demand, useful when evaluating many or large VectorStores. + +**Note:** The Evaluation module is currently in development and its API is subject to change in future releases. --- ## Installation of classifai diff --git a/DEMO/data/fake_soc_eval_queries.csv b/DEMO/data/fake_soc_eval_queries.csv new file mode 100644 index 0000000..a292fce --- /dev/null +++ b/DEMO/data/fake_soc_eval_queries.csv @@ -0,0 +1,49 @@ +text,label +"grows apples and berries in orchards",101 +"raises cows for milk and produces dairy goods",102 +"lays bricks and concrete blocks to build walls",103 +"builds custom wooden furniture and fittings",104 +"installs and repairs wiring in residential buildings",105 +"fixes leaking pipes and drainage systems",106 +"develops and tests software applications",107 +"analyzes datasets to produce business insights",108 +"prepares and reviews financial records for compliance",109 +"teaches students in schools and colleges",110 +"provides direct patient care in hospitals",111 +"prepares meals in a busy restaurant kitchen",112 +"creates visual branding assets and illustrations",113 +"diagnoses and repairs faults in cars and trucks",114 +"captures and edits video content for events",115 +"prepares espresso drinks as a cafe barista",116 +"creates personal workout plans for clients",117 +"organizes archives and historical records",118 +"writes technical manuals for software products",119 +"conducts lab experiments in biology and chemistry",120 +"investigates crimes and gathers forensic evidence",121 +"responds to fires and emergency rescue calls",122 +"operates commercial aircraft for passenger travel",123 +"writes scripts for films and television",124 +"composes original music for performances",125 +"coaches athletes to improve competitive performance",126 +"designs clothing collections and fashion accessories",127 +"helps clients buy and sell residential property",128 +"plans weddings and coordinates vendors",129 +"treats pets and livestock as a veterinarian",130 +"supports families with counseling and social care",131 +"advises businesses on strategy and growth",132 +"manages warehousing and transportation logistics",133 +"tests applications for bugs and usability issues",134 +"builds statistical models to forecast trends",135 +"recruits candidates and manages employee relations",136 +"assists head chefs with kitchen operations",137 +"mixes cocktails and serves drinks at a bar",138 +"plans travel itineraries and bookings for clients",139 +"designs gameplay mechanics for video games",140 +"installs office lighting systems and electrical fixtures",106 +"maintains gas appliances and heating pipework",105 +"builds mobile apps for smartphones and tablets",108 +"analyzes sales and market data for business decisions",109 +"teaches students with special educational needs",111 +"assists with childbirth and postnatal care",112 +"protects company systems from cyber attacks",120 +"organizes entertainment and activities on cruise ships",123 \ No newline at end of file diff --git a/DEMO/evaluation_workflow_demo.ipynb b/DEMO/evaluation_workflow_demo.ipynb new file mode 100644 index 0000000..ee6d67d --- /dev/null +++ b/DEMO/evaluation_workflow_demo.ipynb @@ -0,0 +1,382 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ClassifAI Evaluation Module - Overview and Usage\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Evaluation module provies a toolkit to evaluate the performance of VectorStores in a mulit-class, single-label classification setting. Provided the user has:\n", + "\n", + "- a constructed VectorStore (or multiple VectorStores) built from historically labeled (or similar) data;\n", + "- a held out collection of labelled ground truth data, not in the VectorStore;\n", + "\n", + "then this module can be used to evaluate VectorStore peformance, presenting results with a variety of available metrics that can be specified by the user." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-Class Single-Label Evaluation\n", + "\n", + "Currently the evaluation module only evaluates single label predictions meaning that, while ClassifAI is designed to return a ranked list of several semantically similar candidate entries to a provided query sample, only the top result will be considered when comparing the VectorStore result to a ground truth label provided by a user.\n", + "\n", + "[IMAGE HERE OF THE TOP ANSWER BEING COMPARED AND THE OTHER ANSWERS BEING DIREGARDED]\n", + "\n", + "The Evaluation module is currently in development, and in the future its feature set may be extended to include a broader range of evaluation tasks such as multi-class multi-label classification, where potentially multiple labels for a ground truth sample can be compared and evaluated against mutiple ranked candidate predictions of the VectorStore." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For multi-class, single-label evaluation we have implemented several metrics that can be calculated, there names and descriptions are as follows: \n", + "| Metric | Description |\n", + "|------------------|-------------------------------------------------------------------------------------------------|\n", + "| Accuracy | The proportion of correctly predicted labels out of the total number of predictions. |\n", + "| Macro Recall | The average recall calculated independently for each class, treating all classes equally. |\n", + "| Macro Precision | The average precision calculated independently for each class, treating all classes equally. |\n", + "| Macro F1 | The harmonic mean of Macro Precision and Macro Recall, providing a balance between the two. |" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## In this notebook\n", + "\n", + "This notebook will provide a demonstration of the following concepts:\n", + "\n", + "- An introduction to the Evaluation Module and its main class `Evaluation`\n", + "\n", + "- Use of the demo file `fake_soc_eval_queries.csv` which is in the `DEMO/data/` repo folder. This data contains ground truth samples related to the demo knowledgebase CSV file `DEMO/data/fake_soc_dataset.csv`\n", + "\n", + "- Creating several VectorStores using the 'fake_soc_dataset.csv' file, using different amounts of data from the file to create several VectorStores of varying quality.\n", + "\n", + "- Setting up an Evaluation task using the fake queries file and a list of metrics to evaluate.\n", + "\n", + "- Typing this all together, evaluating the VectorStores against the ground truth query file using the evaluation module.\n", + "\n", + "\n", + "See the ClassifAI GitHub repository and `DEMO/README.md` for information on accessing the associated demo datasets needed for this (and other) notebook tutorials.
\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Building VectorStores to Evaluate\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To begin, we're going to create 2 VectorStores from our `fake_soc_daraset.csv` file which contains mock SOC survey responses and their corresponding occupation codes. One VectorStore will be built from the full dataset, and the second one will be built from half the dataset. \n", + "\n", + "Since the second VectorStore will contain only have the training data, we can reason that this lack of coverage will showcase poorer performance against a evaluation dataset that assesses the full coverage of the training data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from classifai.indexers import VectorStore\n", + "from classifai.vectorisers import HuggingFaceVectoriser\n", + "\n", + "# Initialize a vectoriser using a HuggingFace model\n", + "demo_vectoriser = HuggingFaceVectoriser(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n", + "\n", + "# Initialize a vector store using the vectoriser and a CSV file\n", + "demo_vectorstore_full = VectorStore(\n", + " file_name=\"data/fake_soc_dataset.csv\",\n", + " data_type=\"csv\",\n", + " vectoriser=demo_vectoriser,\n", + " output_dir=\"./classifai_temp/full_vectorStore/\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We're now going to modify the `fake_soc_dataset.csv` file to remove half the samples, then build a VectorStore with that reduced dataset. We'll do this by loading in the original data, cutting it in half and saving it to a new CSV file in the code below. Then build another VectorStore like with the code above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Load the CSV file into a DataFrame\n", + "df = pd.read_csv(\"data/fake_soc_dataset.csv\")\n", + "print(df.shape)\n", + "\n", + "# cut the dataframe in half\n", + "half_df = df.iloc[: len(df) // 2]\n", + "print(half_df.shape)\n", + "\n", + "# save back to CSV\n", + "half_df.to_csv(\"data/fake_soc_dataset_half.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# now building the vector store with the half dataset, same as before but changing the file path\n", + "demo_vectorstore_half = VectorStore(\n", + " file_name=\"data/fake_soc_dataset_half.csv\",\n", + " data_type=\"csv\",\n", + " vectoriser=demo_vectoriser,\n", + " output_dir=\"./classifai_temp/half_vectorStore/\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the code cells in this section ran succesfully then we now have 2 VectorStores we can use to evaluate with the Evaluation module." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating an Evaluation Object" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "ClassifAI provides an `Evaluation` class (in the Evaluation module) which can be used to load a collection of ground truth queries and specify metrics to use. The evaluator constructor accepts 4 arguments:\n", + "\n", + "- A pandas dataframe with `['text', 'label']` columns/headers both of type string, which represent the text sample queries and gold standard ground truth label respectively,\n", + "- A list of evaluation metric names which must be strings correpsonding to one of the current avaialble metrics: `['accuracy', 'macro_recall', 'macro_precision', 'macro_f1']`,\n", + "- A `batch_size` which determines how many samples should be processed at once (smaller size will take longer but be more memory efficient),\n", + "- A boolean argument `save_output` which determines if generated results should be saved to CSV.\n", + "\n", + "\n", + "Calling the constructor with these arguments, along with a boolean to save the results to file, will check that the inputs are valid and suitable for the task:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from classifai.evaluation import Evaluation\n", + "\n", + "# loading our mock ground truth file into a pandas dataframe, setting the type of both columns to string.\n", + "ground_truths = pd.read_csv(\n", + " \"data/fake_soc_eval_queries.csv\", dtype={\"text\": str, \"label\": str}\n", + ") # Load the ground truths from the CSV file\n", + "\n", + "# using the Evaluation class, passing the ground truth DF and metrics we want to evaluate.\n", + "evaluator = Evaluation(\n", + " ground_truths=ground_truths,\n", + " metrics=[\"macro_precision\", \"accuracy\"], # we chose just 2 metrics this time\n", + " batch_size=16,\n", + " save_output=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Running the above code should also present a FutureWarning. This is because at this time the Evaluation module is still in development and may be subject to future breaking changes in later updates.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluating the performance of VectorStores" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the evaluator object instantiates correctly, we can then use the `.evaluate()` method which takes VectorStores and some corresponding names to evaluate against the ground truth. \n", + "\n", + "For each VectorStore passed to it, the `evaluate()` method will:\n", + "\n", + "1. Trigger the VectorStore to perform search over each of the queries in the grount truth dataset, obtaining 1 result for each,\n", + "\n", + "2. The results will be collected and combined with the ground truth labels,\n", + "\n", + "3. The predictions and ground truths will be used to calculate each of the metrics specified by the user in the constructor.\n", + "\n", + "4. The evaluate method returns the results as a dataframe object and saves the results to a file inf the user set constructor argument `save_ouput=True`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = evaluator.evaluate(\n", + " vectorstores=[demo_vectorstore_full, demo_vectorstore_half],\n", + " vectorstore_names=[\"full data vectorstore\", \"half data vectorstore\"],\n", + " output_file=\"./classifai_temp/demo_eval_results.csv\", # leaving this line blank will save the results to\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results object is a dataframe with provided VectorStore names as the row indexes, and each column is associated with a give metric. We should also see the results have been saved to a CSV file in the specified directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Following through this demo correctly, the results should show that a VectorStore containing only half the available labelled data sees a significant drop in performance on the ground truth dataset compared to the VectorStore using all available data, according to the chosen metrics." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Efficient VectorStore Loading and other settings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We've already demonstrated the core featrures of the Evaluation module. This final section will show some additional implemeted features including:\n", + "\n", + "- Efficient ways to load VectorStores to avoid memory issues when using many and/or large VectorStores,\n", + "- More of the available metrics,\n", + "- Saving results to file without a specified filename." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In some cases, the user may want to evaluate many VectorStores in a single evaluation run, but more VectorStores require more memory. It may not be possible to load all VectorStores into memory at the same time. For this reason, the `vectorstores` parameter of `Evaluation.evaluate()` accepts callable functions that return VectorStores as well as instantiated VectorStores.\n", + "\n", + "With this design, the user can write functions that will load VectorStores into memory at the time of the their evaluation. Once the evaluation is complete, the VectorStore will be dropped from memory, and memory is managed more efficiently." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# writing a function that will build/load the larger of our 2 VectorStores into memory when the function is called.\n", + "# you can also use the VectorStore.from_filespace method to load the VectorStore from the filespace if it has already been built and saved to disk.\n", + "def load_largest_vectorStore():\n", + " efficient_vectoriser = HuggingFaceVectoriser(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n", + " efficient_vectorstore = VectorStore(\n", + " file_name=\"data/fake_soc_dataset.csv\",\n", + " data_type=\"csv\",\n", + " vectoriser=efficient_vectoriser,\n", + " output_dir=\"./classifai_temp/efficient_vectorStore/\",\n", + " overwrite=True,\n", + " )\n", + "\n", + " return efficient_vectorstore # these functions must return only the VectorStore object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "second_evaluator = Evaluation(\n", + " ground_truths=ground_truths,\n", + " metrics=[\n", + " \"accuracy\",\n", + " \"macro_precision\",\n", + " \"macro_recall\",\n", + " \"macro_f1\",\n", + " ], # this is all the available metrics, we can use them all at the same time.\n", + " batch_size=16,\n", + " save_output=True, # setting to False, only the results will be returned, not saved to a file.\n", + ")\n", + "\n", + "\n", + "second_results = second_evaluator.evaluate(\n", + " vectorstores=[\n", + " load_largest_vectorStore,\n", + " demo_vectorstore_half,\n", + " ], # passing the function itself, not calling it. We can also pass one of our existing in memory vectorstores as well.\n", + " vectorstore_names=[\"efficiently loaded vectorstore\", \"half data vectorstore\"],\n", + ")\n", + "\n", + "\n", + "second_results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Thats It!\n", + "\n", + "The results (this time with more metrics) from the last evaluation run should be saved to the file in the current directory as `evaluation_results.csv`, since we didn't specify a file name this time to save the resutls to. Remember you can disable writing results to file by setting `save_output=False` in the Evaluation constructor.\n", + "\n", + "Finally, one more reminder that this module is still in development and may be subject to breaking changes in the future.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "classifai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 3d153442c382ca30bb01d88134be4b9e3a767eaf Mon Sep 17 00:00:00 2001 From: Erlend Frayling <194791647+frayle-ons@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:47:16 +0100 Subject: [PATCH 2/8] added diagram for eva demo notebook --- DEMO/files/eval_top_1_diagram.png | Bin 0 -> 55626 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 DEMO/files/eval_top_1_diagram.png diff --git a/DEMO/files/eval_top_1_diagram.png b/DEMO/files/eval_top_1_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..4db8fdd63401baecd5b6fcd9bd8fd98e2960840d GIT binary patch literal 55626 zcmdqI_fwNi)bJ0Y)KI0j&_sIggeF~S3QF%)LXY%7RHT=PNN*yFBE5q&si8*%3B3vt z=`8^g;ET_F-_LyifOp;>-kr&0uIy~)+Ouc(oIU4r;!TWnDal#M2?z)%_4TwK5fI!0 z5)cp?kP=`2a_{YM#`PB=t`}QJxd4y0o}8| zAHqd9UseRA8-Zs2?foNF{KmVw& zyXMWObE-`X#aGlDl3<3g*zmloiD~`K2$6tJF>foN@qA?d(j~)88}!uP)DYs~Iq~a8 zopBmkZ1hPXy!_7WY5{uG*&er%j8k&>rC{m|n^WYSBj-1?pKyHShFd<~h)DJW#wsFwbKT~g|4eqA z=%ZL_HUv@1{+Rd2TiLGtk&1=QYhgz=|L(`b7*Ev#BIZR4$%nMNx?;%orKMLeK+sTz zh}%6@WCzb4bknxbuz?yFFFl&da;T;IDi=kI>M223ZsJ}KL#0AKgl==}`scE9MdSKu zl5xG>y`1s#hVSOorbf|n)ADN%fRqwlnPevUC<(K5)r!UDzIL-I2`%0j+w5ELUe{^B zoN|au$iQ>>y8irKh{yq1W}z2aZ?34dcBbY z#W&eBd5B-yOqo}7i^p8}+~BQ5LYA77x5T}D><|QFw~0W+X}y6`r1e#fl$zEMv9c+j zMW0=k<$R$6B4kc}P+>@}>Mvdhv^OWBV;>KK?Tv0$t*O8Zz5KwTaI=a2(yh=l%Psy)!l&VFdalC$pi5zLtP*)x#-=h6wL~)&kdW zQrsC8?!er(-R7n81jml|%AWy=)Wqox=FW@yId>adH@dq);>pCX!_!;eCe3sEL+UA#SfY&;#~>y@`L z$#V{)_jcvMze1;=b%cxgsCpI0d?+P6ziKdLC)`F>RlMAEVZJbdVM{(+BBiSe>+kll zEo8ARm%NTlwgwN1YI1b;+7CmMx%MMO5ap|9j(WP=!$0KG_~lMEPF9DDc>Vd8*BU|g znxG)I`pVg?2AbovJ(w(Zz99HEBqN^E(X#kQEg_^>>E}qhYn8hlQO!a8Zt1_r*PFS& zb8T$-(ryW`vd~N_mXxkOqIxwQ3{(WA${17k>s@ck0&`LNE(JUrzH1h)>}k;Imt3`)74qRYhfGKwj$qjGL$iaAtgvWTawrU;57}!cI`&|0RK2!6V4g};1-|<-Bq3p!!1BpW1E*?803bgY=-pk^fCJMrwNvQ$x9)d zg^k92QmZY9F&DlUfIYCRtKC} zbZSwzRh}I^-Y6LDoDu1CM2KxzZA6CH@&30i^4II4&Yu4jnF!Ce@&h<3kjAM2*cA21 za-TvRsEOsg`g0a8Y3q8XjGQMtiOnu0%$yw7w>E4|zfTd2=(W6=p?Cm(67-YaCcn?E z`%=Qsg`i(*gG*4vJwp-)`9+|l$SvGJ_P||I1YeeJrw$p~zjj6q zo{mmJt(ESpVlf+K)DMD)zD4R;g404M53b*{n(YR0M+dP9-9Ii)>U`0D&2uM$K~9_&GOw>IJl`Es8_lt{j; z1iFAdrS?zQ{jL8~l+0^INs5@8v5Emix8>%~j=y11sI|X`iAP)lSr+r|r~N6fzBpF# z#|4(~vquT-Imqx!&Cerkb57j80<679m0%5mD6hqp$hWfF6K}R$b`+5>yzwmZ5e15z z#5U2zwuTEkzlY1O|BiSD%a5CNVSyAv?u!PC8cvZO-z}3>C^HX;uB3QUE{p95K`FTm z46X5}h>V(KAqK(UAtvw36XFB%ANZi3c?#8DJ(xokb8sscOZ_MIypPvn53d_u*wy`Z zbsn)7`O676RJP*IMK@RF&dh&3l3yIr@LTMf z5_7%TD{%KLcQAPvi31>k`F9AmENcP;cI#N!BY_ytyU)2VJ&5Q~6nu_fMJ(A|-08pj z?qR;?ZmIewOvR`Q+Mgw{i>qhIRIvc(+uSX4vEI1**EV;tVfU_`vr#F)h+X98v~Vh2 z0JStW4K1QnL<|HO9Rf^_ff%w>hgyen|+Kbkp-s+vbip)D_=`R)XRC$q1Ard5>c zExPuiTwHOB8lO}+VZ1TEUK!5qlL2;mG|G2I-WN-pE~a%0vH`n zy9reWT9L}W0LNLiDNz-6=i6pZr>;aSMrbac?P0hq6^lVFKX;F0jIURQVYWAF2|8uM zi@8G205V_PCb_{am*`sOyQ&I#$3EAVi8B#U8anPASmVOn9Uz#v9VJdBb`O)a4Q{5Y zgh$ZU>ia$Mx1TnCSJ8r~t+`+d+oeV!Uqpg(O~W%{(OcVJnE{Yx4kxwT#+KI3!&>~P z`)@ZQLuxe)1tsgR2_Cc%P#!KQOGv5)yPTIRL->tpWdWmoHw*&V`0gsj!OrK->OGV4qh{LsjKX=FK^LP}$FcUK7Y*KG_kjb^%|^RO z{5rV$*{Dvv&!Z)SHPxkvopraX)!Qjf@zTg`dp%My9}|))&@(orM)0QHusX* zgFl~YS2?^&@L|0K=jJpaN?~rJL$=bohdumSq(1GnkI%#4e0)wn`z!8lBim6;J+a~- zC(Oca_|J|MLN@!J7rxt)ex3b>xz0n)&`mnGt3mrv%gc=>-x!SxS0yF+vNpoNjc2LT~N~) zRO!YvsdfMUmzfk5<2K^T)_O(Nh=#AxqOR7no}Uz5Va^G0|QUHq>K$Ho03 zkN*F6(W@r>*QHl!aw_n@qgw8*{GT0kV|8^E-U39o)Yr?fD+l$#8X0e z7QB1E$BQ>qZqw4P6bLEC7it_F9KWaXePqiy^9 zeps_XeLUd4p5Fh;YUce|>%U^x()z!9%^%?ZXaOp@J{vepZg>qmf^W? zAt5O_l}<>=O&(s}eCwOtJ@;toL&8?H9ZX1h@=l_oV-&0vBL0d6!Z)BIng|L~Qh2o$r}fzqA&Jrw{GP(098;_d=MfbhRk?${ zsj#=TPg;ddfKX6VYx;dZfcHFCM@#o*bFLU5$8-m0US~@^0|opdmGho;8M69!apiCI z_VpE}`W6D#=;&5yIXEKz*gwzrJ=f595jWS$@PJXYD?=sLPeg&;b*wCf;?Jbk_-B(3 zA4}Y%Ysu9e)LAh;cJbMD=G-NZIBX`%<+}4DKtEqP#RZ)1J03Ort#n&f{!IU~9q#1h zMC?kxTrN!ZfZzDrVZJWSZzKI?4~u91lch_fvcilIeg#**FUqj{c}H}w6THEUPUowl zpDniT$$Sasyq8Be;wAs`<;(mFL(MOyZiI+O)lX+YA{OnG>8$nYQGeEHdY%2ng>O+W zv9)cSMGBgIrs?kPo`*>Fqh`KIfk5*rCt5GYq+)|BdyW+;`~|pwf`Ug$HR1!({WUJa z1Q>4b+C&P*sZ`&p<40--jNkW&^mnGGq*S*e_I>~Sx%y2K5}rGXcb4o6ySlov6e0W> zqSi5NzHPL92@J!Jz7Hecy?S+1OIus9P<95dq4AGs$1F7#z6OR5Unm$K4Ex-_FLh5- zbHTJ$n4pWBa9RQD3q9^4IrUto!Ji{63{Ks7N{kAlA6sAh!9Q6AY-Q*OXYMaDdz3hx zB&CLIjUuZRKhn?IjNg1n_=!KF=b%cTmKXxG&uE)fhJCZy${xT1uVcV znJZY;%Zar&Q{Bs#8X>KFM`Vq1!i>0*ikY%%cmLSmdkx>8BDspK7s`D1xF?5HAf_v+j&?DHEFkz z{#g!PKf9f;0_Grg9ut4)-jXNx&9j)+_+1cAdIymSF(gWCNh4^6 zoJsvtx&))@*Ia3s>9^lLWWJXl$_?;K`Iv8_17P=J$v259@oE|4GUZLc4Zr3U5F!4# zCD(h0N%mgfBkj>GFX^BHHC4Bx)A~u7n~J#fXAYF-?I;S~l0A*GEpg2=4|io+ZId#M z+cuN+v_njZQW{cN-buw66ZFX;&$RM_WLG0L$otwD3Mtfj-@43|rN*VCtJmqAm%gQaP)Fw3j}bMv zg;64-Xm@us#i%>QaBIN&P>CGxu$c6J5njI5v9bN}vsE@m=ENG!bLaiS;qT`K=<%JI z*m2AoH;*45$o@R8io5?9yz+)I^WFU!9)5!d4?uArA?^-`NmtdL6F#rBm~?BhNROgb zxg?3drDxIl0Pw3 zNwfmmkC)#tFTs^9h^O3yj2=#9PLK^hP&z6<8NiZjZkNA0v7`VJ`S~5u;Z+?O6Q)|o zZ41NcGi~Q@Cg;d$CO^`!00n0p&G^v-iu*(hX%p3i<05bN(QBU_exzl&v2D{ieP)pL zvV|hUX<}DX!0bJMT=%9U`pZ*hso%;5d7Zat#{!w@Sz~^z3-JhXQBk-X1*t;ZoHr91 z6!O>;9(_#UoouPVsHS|D1?L7VHC`R9VH8-tKoe ze3>lpcn4}GYJO8c9OXQa*?!QsY+vJX7BG?rB1}V9XqSFD@v}SDfL>^bALCYxMXY3i z6d4t|2Br7)W$KtG#|t z|1|hmV#KbUO<(_Dy7P)F!Mq|dst(rQ8qulk8SUmVQ?2AAKiXqn2#vd9J6Fz7LVJuI z!z7+!Jhd_^URa?S`uZ)Mzi*8$X0<(|%n;8_e0pWno27R7t=mx7q&U*~Z9JJ=SIRGO zsQ9!Mobl6v%Ioy%f?>BiP*8O!*`;6Q!RsGP%U_0b-}!y6Oa5j;FUlBA32}28-&puO z*)00yR+T2Q(%0!vqq9e7C}cZZmQH2qiur29D(0s!l;sVv%j|i+Yh!q=s8hFr7aq|k%Y)PsH3rHWE3lZ%*j>)g=dCGq=XL3 zJ8aHdX2}KKxIA^vgXH%Y*%ikK?AUzTLML^R>|whE++j@OSyAzjC&gyzv!8AF$%Q{9 z-78#-kPQ35E@BO0y22}ywVFDaGzA4}5#z2jTn<+IveaN5g5uV0+lU&^pI@%6te^OL z;eWZc!OFTrHWzgVg4H!Ae}}6`Ek@xOr|4orId{+obTi1u>;8!NR5^EiF;3EPS{-l^ zA?9i0FRpstrWOW*-L3W~?l}9SEV2+pkjrS?|D3y%r{gp_ zTmm#T%1_qN8bmb?Fvn-1W#S!Y#~d=E{>(ij5mDJ?ScTjJ+a!SL>e`y&_dlU)nGr~J zDcu<<1!8*tp_nKw1W+-q-P+t~+5Mv059SZTUS^(d9NBqTJrj?C9?5Q0&ZMMEUw&)8 zVpMj022?$57B6Vs3ShUB<#4_`)=lef&y>WL@Pe1W7&@#xjxDP?7>Y1o{eB3OgpQbP zCmL(3drGZR0o9}3IL2G|K80+TJc*-nsT<>)n0$M%T~lwy$2ptaJ8-l=AiTGh#)&NQ zIabBXEL)4Hnvv#nh7bhaXV4iB7~SulQ}G+V^EfpFIyuQ{=xjpQNe!dEQUhFsbN{iu zAU%&fyR{hhNA5#;ew_F@<+npyGSVO;?MFW%y%u2f6+V%~g(snG=94M?d9Z%x+3piP z3i1Gq_90t`IP;|R{2{bze|`>Yk^))0&I3@O&jHCg@jSG8hWh;l5hSC>Z$G$v@Qa)m zGq?(;FR6Uc{gb-wCGtG-xw6%h{@TILHMk|)_v#CkX7ntho(g-M@NOhwd#g&!YmPd} zWAuT*Z2j$y({r}A0AFSH1w63dupn`|m~bNs>#@!46<9eV%n2ndjKIrTnE|@H1v{P3 z|Fr8(w-WYrUf!6HD_O2&3M3cfZ;#$Ia4m_~N63cZ3&Eq@yx~YNTJDV=HQO4V!_RC{|X|d!W;r1$-5|! z3nc5lKh<+0->BJeKmGfM$Kf{P^>kCBxlxJ_8o6FyY$ys%wa{eB_{0s#dak}a0`tg7 za>i0|Xyt4I9v$8$wga$GbAm#Cgh$KEF>K&%DEp^TP6?1bFCK}31`fN{4I(K8z<6{5 z7hj}u40bDCUiFmG|08Sh(QV^*wo&AyRFt{~X3a--6zVR|XzAlqv{K2@#Yc%%%d;`s zlA*o9r~dP>IL&GZJK)l$(7Y6J@8QnNjpkq7IiJni$yiZB(ckKCqNRDM!W*l4w?pSl zI;?Kn{3xgTBk%L#l89t=X3m;B@`zl-x{hTw-_w~Czl5(LaE^IB?J0%vMn4QG!2g9}pgf}X^{%k`NoKa<{nE0p+(W3JjC zVfW$Itz7XbuYl}~x6dzd)LeqM^o$f<^2U#$F1!04$_R~|_3s$~lGIbXY1O|yWLKht zpPurT#5k{XN7Dd^pAyE2J-SIdc5a~}wCMqQ^CQI(dwz@gI2721Jz&aeJeLqMSE61) zym>ux-_p(=!}3;3SGPE>j&m`5k(@!u#Yd5&BH$OO{rz#Y3$mzwc$sP*& zJJeUhdc2?)`oQDe38smuJMRP1h4da6X}6c{B93f1)$`wIOn-F}(*ayiEdiKc6(aDl z?IcdLxqg#^#iUxgitAf%9Zg&$Co7jonarx5?=KUWPDm*Tutpba`iY44wdAvx*GhU2 z46E{!zaT8tqa@XE(c*h~BhCM9!^nJ;zu#cJ%=OK~cs}pwHS<OxVQ-yG2Qup9#e*$A2f!_eysK-gcs>WEcP zyW11ifo1k2QQXURKwrFJ?PRjXa^CaP;}c33-Ly-{b`iSK{h+94!FXj;d9GM*nN}qK zgqj$JqwHW&xVcs$s=-sGfbH<7VO44lB-5edcvaQG`Kq*xy2vUyM}XaJk)%R@I?&F;%2VViUg8;4x7` zc^xBy)F=H*BmCYjKg-H^`HhFcPk(}6AhniO?dp`MM8qPN_kc{H-d(!?VxPqhy5b6z z4IroMPAE30S=F1Z=>pKvGSj_q*2OwbOc9NZGwJjN^qJHeSl;~j!3mMEqo=_5%0Z%e zE~#fwu<|*V)X+^YS8$USR`o(+zs&gHbV0so!8P-}_F4|<$~$S*jyh=xM>S8t4UlhM zzbX&{kfvbo881GFOMIYy?;iV0IF2R%GGIM}&_?YW+f%#2@2w6~J3z%bTV8tu@|g z$4`Ks7rb*48|W@(sQ{j!zB|0%BfMB1i#*FvxB8<|zSPE{Yx2$Icz~9f?+bagGh+Lb2qx-<`5XFdOwRL#Yc2_9OV(iJg0=br=TVkRk27R3uiTIwyvS&c=sTC@rrZmAwzUNc>)YpH`$m zfqaSjilJEd;#KMznn#tJOi)mLbBOZ$pVge{I)B>Bd=Lf2hCw%6li0a?`-C)wRQGkE z_zqqQ$jRx`@oSD%8(&+P{(_^HS5%f^&_7n9KG+;|Xu$+p{3u-zy5gN=-@wFV7b;xY z@tO6$c_*p4d{vAJ_^QYLw!8D-ydX2^w{=osCmF4zq(W_|7%}vK_WVkMIG6fPu~tp@ zSY&LGL9&p5eeI2$rgr*t@;=9rYM{g)Was`wu>E`Ys>QJ_6z|WLq(`Qvuk-Spa_p_k z+q3fWygY%dBA@EDMy}jtVyJ-ev}zY|OP@qF4#G0uSuT8ui%-y48Q@}{ZDlpR%znAf zhE=LkiJ^DrDBpbiS}{C(M-_ItjC;w$KYFYM&%7A{I?`RkmI z&O$)60W1!qM6M+@HHu>co}4lFAjW!AogWI=frwvlKP<6?t;Er2F^DA$eABd0@@2iu zt^??6jFScjg=1sZvsBi_@e)peMh-G{C#RR8YV67zU-?@i6J-+=MHLk(`-CTZ)^}J} zzRy}RpI^Y475}&&L-&aAvcW|6r#zA}47XPHzxHK;LRlF{aMWs)oUdvoGFL1-WbR2Z z6ATl>E5%rB9@LCLWab(E(Yph`Cn-UOh&)tm|685*uoDv{ewx-M@7{jVIe!63N`^l& zh3FLH?+1YJ-#I)?8UXh*%3dbi;jIo_zk$c?by|E?WJEW${}k-WHL36*XB^9k zjTr3v{xyQlOz<(O##6!R4oAU_s+m};Wa}dB3n>ew9s3cx4qyXYA-vd{X}K!?h{kS{ z2Xr_c<>9ZP{pwaM57$#QIUk$EkHS*~cfPzou4%VuSSRqbhbrde+uuej;w&pY1mr}r z*Yk!;Eoyja4d<3#j9~+{8s0cgH+gexh+DFQzg ziTwuX25?aAv|t$*9VK~i)HOBfY>~mzk+CP;;SLo?1}gimEU-xgA9(Wd-B}svA{q;K zRio zoBE6VaYiA(J97Q(Qb}J$-(3B^XlT*mMV7Ix*Yy}>ANvMD<#hIPCadn*wIV)!{^-S^ zp1Nec=L9`B3Cd~qz369~&RerG7143uWXi37l?RLaIS!Y7uQy^atFYssEhUi%{-ZTw zT}wXV(HdpuS<{Zhvz>nuO3pAagiD+Hp6FgDOqM> z`G*s_c!w2)3w2%8;4-rf{{)y_9;@5f;8{wk`X$7;xff-yb+Ud&*O?reA1Rz&y9?no zxvzg2km{HjJbWu+N6)>x5li?&L*0`Hv$Gf-5xYi9&l>;e&z;lf{k%(3~#&D-BzX&1R{N(s=@{whZ>G}cVH z;)_(KTfo4Sl3VXh4t@xjzNfiiL2Sgr@gN1UYuoCluq%*q>t?Y>sg%G&z9E$*!RM2F zp8&y5OGP0Q(Dqk#mptQ+3K?(WRFs2wS9^Fh zR{aKfrcm-~G;a&bAVT3AfF#FJdb=ExJml)-v1uFrW7X=-Z7OG6WxGPo8@I3IexnX) zmri52nX?hnXAy26*pvIlyu)8JGDbb>C@{je0u!k)Mk<*iK3C92SEsVm67&_qE3Cw}i09YCtD6erqBbsh2a ze=BIoob-TtXC^_V=5N>)lz?dVrWkW}$gGtK`hKRn)8R zO9ZP*pRR`m#)LrG7jkM%hXueZ#dS;Dt)pdi+EmakR548cWp~cJiXfyiI#=p*Y3?9=m6V7mLA95 zlX~=H9=E(m8s+4}`^}OG3$LL^;ocF?m zuh7W?iA~%oln}0^Q>Cloumj%hxV7U>TxE;pYn`t#gBU^%55Nf0#yFde7lG}^80IBA_b;9_wv?fCTHc^sh%}5ZQD!~ph`nKzb;YZ*$?p}t8|~X|ncFx4UJ>D#R=k&>O>?y+?8$$Kc&h0`u5LsfVGqgBAEt;^uoN{Ad4o`c zRDU6H57!Y}r7PM)i3~O2ny6lZpx zo^i^TN7*e%0Y$ewlmuj9S5Ek;1;7 zAh!>iXdeN3j%fMTdG8zgiCfHaL65>9Po~~@U91j?71-{!EAmDLcw=5@cZzS}PJ+pYb%*#PvOIoZ0T zpHjd{c+LMhKDnl6FnzdbU*Dzv&^(W!n60meZ~AvXUMQzaxwGI^57g>xGf+F`O@TxB zgtU~xkfd_j@M~n>FtmE=lRSoc$nAVVdC7GwU%cZSUXSn?J}=6x>VBRI%dWj+mxx>l zKhj_SR?8+6_7{H_ZqGYAH!^QHWXTDyMGxh=8$Ox(b1^7r8ouMNHY?-p;7JgE7|a1V zn^Us+(3ma%KqXO?3O27qs=O*s=a-ee2+d3~AvfGhBY`mJD-vWj1 zF~m5~5tcoVT2g~4U(Akqw#;CWe*x+#6jzos8*{MU4P zdwZKdB{aQ#OW%t?6kh`j*E?6L>sC2B;c&ZNHSLXWS_^=E1ALjALB zO4;4E645YFXkobLlw`>7@vGS5{ipujP$Pa+4g6PmQx9MS5FX*(>~Iurq0bGGJ5oqPX)Z8Q|P zcGZS2pQ!RL*REusLq`M%KCW-c<(|y0xk(tvafR<*T)L@h+U!XkQVn=z2NarnPruxn zm6M;?>2Nt|cR{VS7>297MjyCiL5K6z<=e8xL6)sBIh&0$hi0D{lQ+!tv*7rUv1V#a zEV=!S|P+RdbZK9B0SBQnukD+`%j=9hvxe%>Ox!3KwP*2uZ2_S zb-}`69$xZWWdGq3@gC3!U!!&?^Tkwj0@eh1*|`&Xnfn1bv0ZZ|=8A56u&CfH3rtH( z%NMT|6tJrQ9G(#3%+lB2|K39KB0%H~3-Jj6aU7+y>oCKq;772sLQ8s`w>D144_&UD z;M?Utq2h*wh?P((XEA51t1PFflmnnEW(>eOa(6=Xf+EE>qmAg3bt^{3{xt~vdFJ_u z^89{-A=`&jB~|;b9NVTUHLZ(i_~HxT469Q?Q5L0V8@6xh02lDZb(iB^XhucVS`)$p zEFZR4*SS2z)JnaGaBEs`_h~gm(jW_T_jV2xyaYCdxlT4Bs0Qoi&pUpcy%Jx<_MyAkG%A#P9;O&35%57zi$_j2^ekT;vwWbUht?-DL-4A64a7qBHGOR=v?o zDK>aCKh2sNloPSrZtOBVdUW38S|vWf6AIr$abCpc0K*4}Q|cp-9k*W&xaVU-f;|10 zZepb6l)2wt7z}S8seU@0D_97QXml$SHI0U!Az8MM#?wQ?Fn3Dk;^l`?poM1>JH5FP zRny@M0q7D|xO*lmIz3oz=qnKM)l>H%F(LSOkGL4%&Z#a(l#sEP+ct~YTj9;Hk#&`; zcNO#aSlVQaepczsJDhd>?NG%e zcKwoFcSfx{`sXL=&xS|ZRn)x1jJ$aEA^?r^UNd1%nZ!JAf={D}VFJ|+BbLQ8k&OHO zK;Y@xE%1E^%jIboP*O{4j6OL?l!xchH!bZ~+2I(=EL%DDFOO$}{ic55b2b*IK3C&! zARD6|NU^QeJo+D{*+dC~Mdf739}taCU_5B^!hn19hQS2*JNJ=Dq6$#)SkP#uro zkt{p{xhZY&5etD(+JV5(y&J(+rbX3PNKmXIzOKj2B&zJBJjK;NLZaEa1?P zMK6I3mM3l`@Z_$ND#(~g@(Z!1J(gb&>yE%JKn_$#v@i(Ez#ir_FeZ*Kf7-S6rr}M9 zNV(+PT3ONVN284Nqk+IVnw$pV&-&0pjKmjY>;g6kI}8Nyyiq>&E#weLQTrn4b$rq?ialvk`I>_aKcBu} zrA?LA`0M!S5jsaD=HqN+kBy=`2@)pa9?f2E zGk^Fc3&Z8xhFWe^7w$k*@>qn07mVidfH`864xgqSb_|%K#`i zMI};9)m@&l@gw1ZEdB+&P&UsubA=k#G3|=~S$`~BcvUj@wCzBbt(Cp0FXA}%Dsl#O z)Q>tpcjV`t=eGyLzWs7nD4g4Id-5bm72CtKcb4sqyp2Noh;Pe-50F|1H5<{f}_y8g|U8|+CrFT`h=m)bNrrLq4P#v_ZaITq%VQ3+9}b zhrSQ6UB#?WtKf5=6P8shEUfP!YfpS324=59MXzl6d@_;4Zx#HZ2}wgj%C%orFbkpR z2YWfdU#C&5nU4D-XuP#RnJem2eK~MuB9cTMdlB)Qe5Gnd2ydP)X5#i z|G5$95%#b?!xtyOxZIAxVpkq6o#FVkG#II=VerJ#>SQ@<#lsWew%`5~EHEd=ZN#L;d#c!tyuGO4z<5yihMThqVTHm=MyrwfvtaHH zA}u>G+1LQFdPj*~?_}H;_wz=Df%JSdi%gA(R~d;|&-3)E6)@Y+3ZHu)%U97{_F(Wx z-IBE`nmVumpZ++=H8316pVI{#&OHiUqprknZ*(LvtBtn1^mkmkmMg;X<3ky@u}hVS zFp{v#_5ql(#bO@9wYeM%ZdQfOp~3CQ_z21Uo`E(bl;IsH zqZ5$Zab)e>2?&PCsYOpg*hIu*I`DKGwwM`FdHX$A+Cde}(oLlWs{sro23j%fGmPqN za#3@Sd-3+7Y4|dCZ4c6gg5oS!eXYwK2a!@pq&Y6T*W?Q#4V(Qm1MS{a1w&pOV6!tU zEoaUyFE5Vd)r9AcPJF4s*CnMVmjLYDVn}yj4Z}_FkJ;LF%++70QG!}G4bHwhQ1zSE0#GqWnhEe-)#YnOhU<`5*QXIaS*B5vM zSBos#pXZOFX@%nhdeC_1G}yCQ=hcIm?|kC^o!152#=BLb_BdUyCwZk(?v??-P!2yf zOG-|`zlBl|0-{~v&R1?s!zt)Qd(!QeUremH=u~^Vwobk?ClZTV6y?DKQhTKH+Ih3x z*jmu3Yx-RKRgCaRZ9&kse}v=uVcS8(HO1)iMEE#oYIH;e*{%|cX%G5pL3M4jDALh^ zNN_JF9DTOd5x@#B1Pza4aC$+0tiyHl%}b5*rwV|5 zf`MMlIpbcAd2z`M@5*0|Vti=M=2l3BQ`V*9hDG>-L4)Z+99l zZg*BKRz8hLZ7GzJU-OCaGQV{>SGt*Vm6rPS!qH4PAnz z56tk3iBUa$`ZO*jMdvzb5dI}kXb)-k6AB8Kq(ArH)|z;RBBll8O!LJ@rG#A8Y{qxq zqh?bUynu(tnBg@quw?pzmJLk};iHmKEi z@8%Lqe>j}y2g|hWd1b;34yhlv+WalG+G1`Dn|ihUbMrWvqbHqzLbE0}4 zzP0RmiB4hmTbNpT)QKvQk9-)?x4!l!!K;2V*NgL5v4I^-$vcVeE+7llhsczh|raJ$@&2yIRuD5ok z`M?c&ODg-81$wx*w%5M4E>A|N zMOr$GbjF0uqaB_3bHu690s8_x0ge0YMX>qfG?JJheLz<2P!KHDTO{(L^ z-z7|(VX=Tsi{V01&Qp2vn#NA_H;r=Hp(lI*PD|_8!?nI(VhEt!w{uqZyJcior*rK~ z+}IlOcgeCO_Ma%HqBg2QrNv$YbA;$%E5$hn=iumR#dJf zFC*E!&%Tos*`U|K*YB#(JK=#SkM<0`k~#;IXp{bA9zBlW<+yz{>~6J!l#CZIL>gr5 z%h&owytvv8^rb139Sb#bz)4?f+N_oeX^{d+I7IxD7bfbpt9L81Kdt}Lp-o9}ebtWO z!_eN;r1V3ZzD-JSO!<*}C2n}{zX8395IN??g3asmd=?2va=F)6WTX5(%n`Cxo42V<-6X;De+9Quj6M&ozfk|-t{>;yE4q#>_kt1r%( z*r3Zza8XlEHTM|OQ1nJ(&XQP)|vTvF+ z>-pvy4(3+bc3%a+C962JmDH^_&%<(zc}DWS*)+X$cE5SUfqLvHXw2L+9%;uRdRR>E z3MH|>3>OK?c(rz#VacHctkB-|m z!HA}0$ioXVWvLv6DeC@v>rV>9*7(&A+9{8&OWln?-Yq`-Ww%tl5}ZseO}*_l*u5{uX`QagV15mw5kIs(izNE<&ut z9HmBLFZ3K_7<09`+{5MKSs}zm+O+X(yBjSlq-Y&tP!ZtPB%dt>T~;!gA_N~bDE|4q z$yAQNlQj+Xv>I$fd#b^R<gL%9k~NCX18%>ux4oe%pJCU%zt$a4 zW40N9eS+RMB9HF&J58yl6cZI!cI6xD%HGHMTinUE`;4g;{a5=wQELu+*;i99My>&@ zi^R-+yP9JL9Yi#6wE3tr12A)96%RGbKBlWhgPo=W$Y{9!MV>3)Q%Qqu?5zU>eq$fq zKQ3RtHO1=E;$!E$5ivd+GQ3ISu`^DppM1QOa-GliD zLu>0P$HL;`1N8~Rmw7fb3tY|A52?aia5;>v2|eoRSo#}H$J@2LSPrYLCyPx1d{}g$ zD{qkXmO@a^vt9qAP?f2Q@F3?ExO&i`all+N)kj7A=(lfi>N@`!x?a$_;IC5K`^0+h z-`SGoOD&uJSc~9}}xr<6{9^gCke*yy-}1nsMC&o}2(gsw9Vi{QTLhFQ&ZH(kz2 zvt41Pu9y`Xc$ni6*77UfbXpG_8>`d)d5C$dV{m9ZJ(etcAiaPu^51TQCMIQWWG_@s zV9*6YA<^e?y5FQKDT7wA>lUvM+tCp+|FJSpMP@v;D`^ zlQk|97Ke^!Z~=QEZ%<~0{WnT=wpIbfRgs@s7jQm)_S`+F?@ustP(Nbi{nQl{?A-bt zun`ft6EjCC1?pP|JZM->JWj9+#=-_2N-7ASrF0<(6`c?M_wj!B)FNO|5ST6A?IJ77Whq)K8;L*r+{ zF6S`u#8U}$rk9J}+G@XHS{tCOcHpZJ%AwBEBjR*nHzS9wW^*Re&P6W6#4_86>+}1W z@1naMm#^?f+{v(v>Rs-uD_Bw+D~JEIG9M|nz>M6=RB++XM_Q(AX2j4%YgRZfvg$4F zmbr+_hrFH&-ODR{U2_d_FmD(~9Hs)<8+c5_K*J81vnDKKTqexkn#kUIB-MJR@o6I` z&OsbG0R42>H&&OE759F*7XMaO-qHy4<-NUuv(EpRT9;DJ@~PFpeqNag!dZ@YFO(Y@r$K5 zO$d>FlGQjkk^x_fVZEjM`?IwAsH!mj?q$2?zwz2S8kuV~bkZ(vm2^j6M!B}SmW$hEm2;h5 zv*=~{(GRjv4m;#LR-Kjw&(F28tef?r(me7jdSoGuske2nbz~7LX3}eHkuHQ+;Fi+qzgrC8CTHq?M+(GQ1YYO=Zo<{Q86lj3kEFzCc&gpifU@LMpeG^ zy*~C?Q!54q^rZlt03l~r`{!ZAA;jk`)+FZ=qx_`$+TN7g?x99o4!A9e!SDLgXl6b{ z@Z#ETLh}CzD+iK*FR(I2z@;6-89j`ZkhWqZg~VN8k`-`gxnp+WtGz@k5QWgW)s&wh zv1=E@CfR5CvMOfnPQ2_a`_lY)w7IRtL<#>)zzUUKxI`1%N;jR}b};_rz_+9s_=W#>pb)QIaP0OA}D0OV1-qOW3@GP`V%*mhA_GAb4eTc z#C3AWPS|+45AB#6<~?6DLLU9_;AggUN0BidCi zMCNA4u5eFcosmX$G3ac0b8vg-5u7Il zPT7b>dYSdqscO4|^^ldmn(3}^$HXdR5OoBkOxv595^@NnQ%s@E6r8_ray{{;MTzvK z5sece=re~6j2mhzE514`#As5di#~8>FPXiXYtz&zc}m=#yHaRbb+OECSf!Cp)@4Vs z_LnJHMfzKR|EInR-#%YyaLY~F?^a(;_n)uE%ihMoQ>jKcIO>JDjD|h%Ep7gVHnh8e z`}e1U+F0^^7aB^HwH5lPOkZVI))j`j>9r+R^Es^#Oy?Uc*s+;M?V0+%*;=J@oIW_a z8+u|@m0fu=c5xD5Kv$;YDEo?&b(N27m?y@d#>nFe^8%Bb@I@8M>{sFsuf~igp8KDw z`RaMb%3Wu^F^YT4TxR(Dx=o<=?Y+n`}L#I1{W!BDCn zd3B~8_KRl0O}W0wF?EE7zJA$1jeXOF`qeko$X3@FJ38kZv3ycp(}y+dTVy`Uo0FL8 z@fsJV(5$=DaY=gv!%TB|v>uPPZ+yBOwp-|Qq#l5?mcEhdVj(Q8aM`7&GF_24zjDW9 zPSqi+F*~67;{yHdMP{&4mrs9YZBSWLYmzy#mt{&*?EELccuk$<=gFSxN+1i8`_&Zp ziN#Q%FS7LS08}lGWIV9YBmyw@XHKJ&M85DXugKnOUU?vR^s>T zE}4yl(U-4_jaVhrM(FZl>Ye4TYMLT`J5j5Eh}!;#^ONOnXtlU3fX+jAXyVfFm@8o% zh3TZ-{Cc0@2Oo@6ctuQ_dPmfx zo>$(?mrlIQvP;xu{dPp;r;JeHwg(zocMT^3++r`STMc|k9z2y$4+ZFD8bvJ`hl@okQ0dc znn9`2fQm(hm9+1s>boDfD=5IjZ~FifUB&v{=sUYsdhN)$%#!3zuC~8_I%0pks{gYR zJ60>umCpIY+#NP1L$c>cdON1Rif-E8jB3(K8^@#@W@@))p;K;|U68jJuqS=;%C1US zpRo8K3#qk&IfaJM>u0*uSnLtB0xE-lzNrv+9fW>2RT0G3o7(IDlLUWRW<<}srcA>W zB1Q8x)eWTcKPu6l4B!}+@+Y}Wa0j2XK^;w?tQJALF9165=mCloyb$?#w4&r83&S_)6UiFCLqoG!V<3%A{ z!DE|GgSU_L4vzPPhY0DkS`3E!kv%5GQwD`qnIQ})Nev@h6U$;|kagYPjrf4fzubDx z&tlew6Olgy8n>tV+o1|nN6{Aot;Hme| z?4DUqe8@YPO)U%Hs@Z(_%KM`Aj~hA{SS~5rJ&s&h$RgW@WxikIFTJM4jR`D6&q-Bf zt!I22 zPrvkYT2h>tvM(gRg}y!_KFjv#tZiERg!@M7XEL9~+xucf&IFrx9fQ4WFZM=7l_g%6 z8nG%9H@lvJFDI{vo7?%8JUgt!iO^3^F>#+Hd{Xegq%q#oq}{x)O45$ki0~^3F;q~R zkfFA(L$70KfhV-yHIZw8WDVrVg{k*^$GZQ%+kS*R6U@m6-@0XeLtv{9$U0=U=Tf^$ zj6`t=zVg9lj{bTzWbAv<0MH$z#Q)8@?3&$dtUaIRm?Ubzo(DXzLqaMsHS(wsn+wQL zZh#@40WUOYPhMfY8Ygu#Cgb|(r}xg~bgV%38&U>kU$tJwG-d7l$XkgNZOiaPZtdg; zNzZ;}#|N^7{#~C|8_ci*HHkFjtEaYvp9oq2UcnNhrk-78IF>rAB)Wi&0NG5w zW8AO$=%Uit$K}-#-z|=Ps@@jJ&hR@)dCOPK)Y))u2|w4E^Xq=O%Y7Q#^M#7zaO}a{ z(;Mw-o35ZqKnSH=L{U`>1+-PtS+xd4pAHb7s6Jmc&bOc3<#V<6RFbD&7d&lWrH0xI zB3^RL8bBCrTx|#O@F@97ukYc)zYHH_k|u!!&Hlv*9$03%hw(HY%v~IhM60}5jJI2C zJh`9S1xr=BLi|do=OzxrGNoJ52Msh*x2cuEq%~v^FJ2nPIG!V0;6?UTp0c;crjVa_ z2c!TXWK(@+hs(RZNw@9J_3L-+I_>JJt!&9bC zc!kHk0b*H58k=~-H#ukhvUjyj>~VZ22W(AbW$M$OL6_M+Z74^vkg@YdgKIo6>`z5RR+Vk90Xd)6M;6?wUNmKOKP%{7VU{x2Z7*=J zvc|y58vBz(NMCIW$$+#hO~&EM4FRD2z2-{ukwf9Hg!D@$O+7F@lg2XOZ0FqQhs^bp z_B>oy;0Na(Mtu4n+_O;k_lr<|Prxi*Gu=l?wHn3A)$ir@7ei{_9-_LnDioSR%{cFd>23t^1d7J>P zVYdafS$0>eo7b$-fT)J$_l31NpJqb$`VZadzxF{>;!p^X{hEK~)#-}O`{q8~(6}6$ zS@7qN85K}5c7}Nl%XWO7Z ze+oW)+>mjW)W#VzwV#p|bk_%GO~`X0>`&L_YfJ0p_VV^U!^U2Qu-CeKzYiFA6EE<* z*0;C7f1X{`q>+jt)7wh>+3k%IzwN(!D@;ToELuuC48v!G>|I7S0Bg|s$hrhjGV=PI z9M}N|Z6u7yBNGlUw|qnc;{F}k^ao{BKDB$;JrTz=<$n{fOttjxL3GRB(#aGWkDjZG zHFRsv--KzY`)8uO$qp-LhtEO6$Nk5WeglFvg>}pVJU<4*W;u>pc;#RjkiE%v1Lv_^ znWv3xcOs-(lWTZ@hiF9IEJl7`y`I=?HN0`W&zPD4GJyuN=-G4sj1w--#!#~w5{O87Yxu815 ziG=hHCq%hHEpcZ?A!ogt5nyss+^nsCaKNLfU$aDC)QDAD_L{ZZE0>VFLhcl8PYYNF zHs?o^WqlW-S*OY0^_q?5U~JVMCdj=tMEc?f?<Ont&6{+LI85Vkv#QpUoW}+s*Ei_v#X;22No{VMrle!j z0qfdZ$CK~OtvyXyU6j2cwF;!>om}Cd(Dxyc*4u7i>@bvySI-42P41Fp*12VB+2u$? z8>Sy+5LZoe-b7Zt7>f%7A{EyrtFV#eSCgBk3f?8Q^N1Azmksl^lMa(h!4uW`rk?o)c%x@^-4K=OcU zp{f4$y$fh{hS9v5vxa2;OMv->#S-esHiXp5N~;9!*j$(UE7yz514`t#$v+jTR{RK`_coE?bo=7vaYcuiAuA8M- z3E&k!&!qowlyDE ze}1}uGzZR@LBCT4XWIoe_@4yufP&!SDi}iBW`G z*w_X7aOi=3fO6f%+8%QB1<>fKjTnub--;AB=IASbu2xu<>>b0qvifAJsD8BF>&2w? zmF)_V-|{qlt_%Gw4k{32YB0}<>n$t&*#NgE!8tgnJ8-6i_ubv)oILnbAoZ!1Sj4EytR*GM3mCmeQq&Vw~ zd*B)?Sk7j-8!Z;ozH#{u1stx$iWh}`;srv-#N_k#oYD>9-S}FlN=6A-h+h8|A zr6x@0cz51Kh~1Xi)k1v(oT!ZkaY77CXV9V^`%0DQd|=l;0h9~6J9&ej-z14yTtnBB zVtR}6H*S0({nfwyz83Es-GiS=`19z-_Cw0{>dvyime~f3Nky^NAAPy%$JBKB(j(8S zsdK-Q`zGQyQc$Y3672MUG)Z{Wl=Y;D3Y#$ zqwirE8MZx8lb-syy$475M|BlgD7#X?%~vv?|0woZ4$s$pJO{GO{Zc$T4nPd>jOq{BO7=|LVPJ)QPd^r^=c_va>%lgLJ4 zxgq3>cd65rRyjQ<+-g^`p>LgCOEoRK)O=1}5py%yn3u!Oq>b146LaBhLZcg~uOXsR z*X)^H8Cw@I;GdhyNiF}}D-@xfLg4-42|!l@r+ zhwGF%{Nj*-z8wkXb&j=?u!tS`{rYXNpNs{k!g@5rBY;4m@ zc)h7uK2Wqe$E&p*2H=fER^mD_no^&$tP+7=-Cnt(D@uKotGAezsvXI&xj4z;dE;R*Ly!SjImAf3}+?!uxRW<$Oz zzY%!Om%{dKjcBITDU2GFO#fD2VWe@~YSh;5nK8t`tWWcM!*R-Y?6xTfk`O6N7rlEs;5!j+0H-$YBvWnFK1kkL~-OSZ1!W}6w z@%msH2?gxO4h42B=e36eY8z}|?bjTccQ=eHLcgXDy!_arliM&lOdEtjbd6fh)IkI7 zIrp!m&hNCNbDZ_wz$B)0U=rNgY5W+VWV3g&WPcL?4M&9GZ!)LZY5Mj?+!t?qg4)E5 zSIt&SXmi!2D)|6Y6F5+_K~(|$KGOcS*C!eYS@tG?bc05!HNTI3qT8MLCx1_JE9)ys zr@v_Zy7ZKqIeJXrbnGiO;LW=}@|&%0^j0PYZ9hhz(bct{P_0&N8&O-=k@bv7Jq zV)6DZ8k-N*3-|cz7cvDT<6O+o4mUSvehVW$21(pFd_I!QbRCC(56sgL?u$ivjRLb# z5`&kI3D4#>VJ2^^p_4CFgt*NS++g7&o#vl6O{bnmWd^vO^u9~fPHL>^k(4bmZJRpX zFj1L(r#7u8h}S<}p5tvgdcgd^V&>ab1ZV99c_L>#UqQ*kXU-WxB8k%BNJ~r(KLlQh zQUgMkC&Q;nX$VG)=3HO;ZtTCn}l)@A-=to%3yxquAUhtbmWFfOvRI z5o`wwqnxO$5hU-(Z74Qi zZD~$5At9mLx4O@?VStQJAhed1TQzXxoGnmL09f=_qZ)dD0RaKRqXWmAf_ZNaM)ZX~ zidx*7`uG_wSGtmY2e_kpZ9sH%lE0-Zr8h^Lspz|r=FTy7U^k0?3wR}v6m2|Y3bw3y z<$FsVekbr?F&^S+2vIuNYx1f4i?rNA&c=G(Iw=ei0bUUJQPdl7L~2Hydlbv+qjSZ> z)t85ht2lq{hVeIhbbYKHBqzh%!FIb-HUH_pM-@6F<8y6I$tRQLJ;mF=sm1l0peq=r zht%xpSJWlw9`~WD<5tb|I+WYLM$j|@MCwiDq^)Wx2 z*ZrSr(+6#vylt$YvoD=_Zy-U5(s_((+b3%R%jw_G6Zd~#IX3S3-=wZCrrM6rSZL=^ z(rYu~{w=tQxA4@yPsYcaiZ0HPKlf}c)MVu{TqF%5fBgl)F5 zQ$Dzx|6M95e5!Fu1R1>czkq9k^PU7R+D{ENFg`|_Meci?e4tun6OiNYX-xX>E;4Wb z6Tc)BbLw}>>zGFq@gCs}Fzcb6WxgTMfmR77w7K!)f0x>({!=M2G5ktuSN5xIo&~(u zt-4qISBJg)cZ%uKDqV~J?-Tp)e^pBozZxwJb&3M+_k9gpW+aM|1~#4Hik}}gm>-uU zKKQqGZ+zkZcMs{S{!ba`!wv&g{515mEDC&}F5ufHx`cPeI^2!VQ2G!eryZWnJV-oBNRw4<4La@}+6#O1j zfli$cHuaO0&=k^6V30R1(4*P&^}1(My}Ev|sz?2)Z8Y77a_SY08C1NYmY<(*4Y0mO zFaX2GO@!w@6%OUsdZH48RQ6RS)Y=eF!WC8=1m9Cfg0GA750U$CB8`1fCmKezcC>PC zvvy-QZ+QRW4I}9#xq(cN#EMhOFsq;d<>?s&hvb;5rnd>HQ-5nydUG2WQd(h8q8gU4 zh2}U|Z6<{O_d$5`X@FDJINyyQFgL+gQs6V$QJEp9O#xs_RpZ{hgbO`1SD9JVdmD%l zwm9@Zv%ldN!eNGR2NFq$&!;F8k^Z|#@X%U4FZ=PNzu?Ur_LpUt-wZm$V{T^q0Y4q8 zt6-t|N!`W%20oBQ5c)ZOp~%AW&0c3I?xr`N&7Uv_vGxM>;~1SLK0Ei2$T%sgwgvrl z^GjJj$dQbTk?JS1O>cnmd|GvH<7&?5D0LxnBx05lLtz#3)A3Qf#uW*TphWWz945ra z`st{;cYd%t2wrs*HHG!4-VCxAH%mxl&QEu7B9HMLc=bQz-fJj>yJ0H0+Pbtm%R*?U z^p?#20Aq#H8sD<`YMt~s} zTX7eoyla8elNZCtc-DuJk#p}6HrpeHlsA8O-2QiQ4e*>-3qA(zxW*j>ab_b(Lcx@i z;#I<*TqA_9KkF)Vt?T zAmHzMU^vg6JD0ZBj-5SLVXZYA<(wsko+r=v)6g<@{Fp7DP5hGa*W$LH7lqv*(aCkP!^NQ{sJDmR=;#u!A{0od2dreY zgyUn*WbJQ3g;r!Y>&Ep8lVc@LDkNVX8ob zo!~oLZl^?X(rp7EDYh_nh2ig;p}%`ip1o+v)8?A{M62}OVz2j>X(fAE{qBvVw<)F- zp;ZCok$~shwop`J=9d@%(astaguQ$#m<_|)PXX%RBoSSDwvmz#zNs>uojfotX*~QY z_qC7DAtPN|YSVY)YMCtC#-V}VpEhP3Y(!d4KewChKi}s$orl~8$lh-+l4Ed14q#jC zvA9Wf%Q%jP~ex8$0>(QCIwZ99a=Orw`4rLy3MSZpZ`lI1lwl~=K3dF0YCs+ZPjPzOC zP7gTVldyU5mkTUL$qs5#=k(pMUfDhYM$f$X71!dFISUXaQ+j5;xaph|6NFIJX7mz^ z1q5zkI*lGRi-d1WLj48^pU~xvV*!gcVIi`NoT|J4)AHOh1?*GmFwH^7N3H72v{Lh& zbF2foX>vnFW<+JSk#feMWcEYMX`+M*ch*Gh{b7PjM!@gDMY#<{&+c@~UKJ*K`g^G_ z&(M zS?z)~fUuY^%FbC#lvnFy&}obYhOa^FyGl5GHW#4~(^>?zwj(E7PLzD6{AR=^Y-`$M zaeG2|_J-ok^MLiKQ)2*9(PjgkXTjkS5+r|(c%o5Z zKXao%D?134PW1PiT6Kj~%NTe526V;23R@1+ZE9?f$7~aR$QMl3(8_2W&n&=)QE0=oj+v_1pzU8()!U9r zm8YTJR%6|BH2~$FhEIxgi_y`;injzTKl=_0tb5{<19D$i#!_RCgd3DeM-3Qxi|1oU zV#-<^Rl(yPvw$q>@=}Sqj3E0kt7u1?NAO1=7tJ1{_}nm;Ksm~=Q&uFMpm*!-!`P*K zyewe$1i2F5-uc5BvUc5Dd@+9Um!?JADN6}?0In6e)vC}KIgR~bE*Fio05z35ySnSpLc%HGl>OY>xTz&ceQ?U?j6EN`#nrrHcv z@p!f$%ICJ33T#cxIT@AD`23p&@Io`vK--I)heX;KJPue&DIO|(Axp@--*=At8Mi%~hPS)jn`++M|?GGQnSJ7;N-72Hgnw zG;a-1H0kTrkMq1&l;DJn8hc(S(P3uYgEwf=2PM8zrvkMhSgdZRbKL7;rq;3-*1UQZ z8Cb6(>4{b8nF5wDkw@(VJeCh^LkSC^iN8x;n|}M~5|G0K!`78dl%EfwIf zS^YaDexNH0Z4t3rc;RRM5tEOCM{|^ykfY5vjw`Gy*Vx5`6X1MM!xf`KsF3nu&@JI^L+lo!R1ej-qwXSJ?%`DiI^6 z{dX~Y9qKvr(M8MBi{)9WOm&gC%gkjy241ID@rXG3%f`u{`8TK0mI;=WboZklOV?HT zLgZU_wGSt9{S-V#`n1kSS#<`tD_g?Vd6!JOrqLD&4){$+nse6UbuM@+W>={ZGlVZQ zukXy%rnr0D=g*fxD(xnPxt%(i0L3={<vL)rmD3E5U%5wnc{c+R2T4W%<~#aqMWtN%j3UXk@A{j5M+*b5@^o5Dyp8iKU&qmY zRsDgazjm>6)us@e2~utDT zIVS|`K_s=Di#%5A8Z*cwZueyk0TXHZgBdw(D(ozY1AZA@st8W!R-T0OcYz!aW~rU( zq@n$XJH2deq4MZ`0&ylY)Q#xBqozeJ0~ft)?=>vhTaV{gN!<>W*ac=cPBDX6Xwrbp z3~&zVt@Qfxjtz+@^Zq8b!r&(`P~;Q!?js9(njd5~Z0qjKk@W?8F^l&UMYg_A z)y2T#1;jaw68nb}BSstht64ze#xn4Xx`*P6a0KeE_Ca+@2Z89*t}m8Rhh2*Q_U)cl zM%4v3K7XoGda<>XRwY!<%*aKGHH}hHc!j9h;RpU42<-RY2e52+#_IBU9~WrNGv= zuW##JU4EYWaJ%(Wzxrq0L}B@Wl>j?w5w`P`Bk-!;)FsqvR#pD`Bd^g*HKN;g!jBw%XlmQ?!xi_?r(s zrheB)`SwOnQ~ho%q-vP*QY~{DrjG=-L&~KE$Bh%9plm?h5gt_L;m`}Sff)$V&mtW7 zd^a%Qrp&z&pM+`~SW2OB-UpH4AD@0~!wB=KpVMp;9u4r8%&+P^CRFw^TDQ4OqYKBq z?H_)MTzG#+ym`Dwx+Nx5??`+`WuB)n`RLMhL}76l&?FH0$ydT5jqO9rH#pe>PWmJ=EV%4?=T<= zup!vkZ%K9k#(z>Ts5_bs!_7dnZ{gSM{4Si%<6N;gRu-6Fv|}67cxZGhVOf_jI89<& z$Wxm=)yS>xZD`a1sT)LYwy_f%ePDpa^Nd55-lVsv9`5i;zaz@KS)r52RNf8Sv%;wR zPCI9*e*^1gVN?@$ueJ;4?d~_1YuNCnJ4hlo6#2`ZW9+IZH?y7d{FQK6Vbq1!#jnRy z3M%trPQ_d$Ah0VEy_J&m#X93-*<=H?P7~=S(8Z_@v$ewQinYahDhIgA`*_e+^S#xp z2OAX6b&%@T^5Cr2-7ZTGh4GFb$?KlOdL9`88rtY%0-Q~4l z4(xS#QB66%TI56{j{>*Lk=~NTWrN=Yu+`{}U4_-;i(%1p?avs&ELpCQ`Rzv$k&w{+ z6zHt&*dadbl2$#7#H((2whtdzY0`>AD+9((MC_>HynE-O{nXl8#s%!}r6+UMD4?}P zKq>}%gx+)^w2bsA%w2=46AJ$k5KD@p0D^|_YcG8#xk^!{%^Q?!fq>}nJ@@Gv9Ul=2=#n!hh>9E z?tRy#{vz5Giy0Q;0Gt~5>lZv|@P_lIOZ=xA6BUE*uP$kk-qpVyQ3ypP-!+U0wY8 zRK=yMpmY`Dhih3xP{O6K9FP-nN5pb0prcFm?Dq|bhJftNpxERC2UnqrL@ ze^4N(w8a(27R$F`e}A@E04nE%eJ~<|AKw%QaO(5#-TBgn1^{<1U_cm|8QFlBy6C8Q z4(r$DYH*FfwsnU~jvH#&Aiyv|tpW1}8ip(;VVVkjs5E_87@0SQ#dnW*7}iFS(axpa z3%o)Rv#`p@0dJ6mW+*Gd1){$5Rp=ZoCWAwM4qna=VzB+WHV{BI;QHu+WfLAqs{G^scYsSv%vz3Y+2~YEcLy^oj6&860=|^YudA!8 zDio^g1?7)#&?Z#m>V1VeA%dl?OV&8wS96K&tM+br?_wz+9j{LiZXVL4UxR(|p{zQT ziCf`adHLOQ=d;G`J}ogaK5onS43{ml;H<4Fv^$DZ;pyNsTMHF*s3OOn_^K-@uL9+A zmK!>AsT6>Fsm*-5P1V@^i8#&_A+0Y91g>nhiu|RGhHweIpEGNGn z3;gY+y-XzDBUV-AG^VIRd6`Zg*|Te9T_MBh}!{e{hJw2Qol zw1-(lGX3EU#hx8tq@z@6nbHTtk2KN4m9t0}dQkf~ChVfx~crM$-_U zpD>n;`>hk<>585C=tq}w4VGh8)i;>Voj;d>mUfjTWRU@`=+Sy7+Z>>&*K*S`m|l@n zyo>5|gPCdn&XVH5n$ebc^)RVNq-lQrl0a)UP_`ur4m)hn&hHc>rmwFZg@z3i^AewW zc5WsCJy+;nuMylkOGF}@ImHPD4fRx8HOQ)-pRC6LiZFTn&DeRX!#_8r7C>K3RRoyalO?w#;7X%WfmrNH_3#GbwIuHd#G zfsu)${jR^QnB-XSuDoYtclvJgdWhB>8Zf?{W_cW(G^Y0N^?MG_Bb~;Fn z3n5?QvJ$r>C0N9+g|AOqxY7{E>)Z-BMEn7fyn^Sgu@P3+m&F$+WEz`QW-Dz{r9GfU zAzdv_DisEnld+r`8TRe>VgwwkK+|73F4}xGCc3*$S95XqD9Vbh-=ABvcI?XV=dilo zxizv9kbEcUeyt|H`D6MePi!oHVV^C~8i0ZPPfilnsx|82Ze_uMSw*Y-U?I|Sai1+H zT{jOCX;V;a0vSvoOEL0DU&+c2qxDl*H-*_na@fIjv;OAeyS|4VyEtE&!A-H(Opkpp zb(%On*hE9#8{k5hRjc^a!4PX_KJR8$IpRKja3e6oZEnaEdOtDkmdehGW*v5!IJEiI zV+e38=jRKCYx2tXbsC=Q0d))Y`pUTMFLxu-$ChuKTKuV0rN-Q4lDl%DWI&scu{U-Y>G!4W{7g)ZC0>+qsl+BEwWF=4CaiV=aMD2>E4$EuC`{OdBX0o2Hb zQkZcS`JGb&nXo%rm;Y2BAPvGOnOjpw5p2d=(+6tTI^D*8zxaa)vop{UqbQ zw3+Q(D<52yHp|K%1jPU8I2*V@Oegp5I@S~N{E+bPjh5PY2_0k^q4vYf#)L&vNmgHC zv8vZKXM?%xczbkv@52p=KReapFvgV|r&)XtYN_dY6wku3)lhph0 z)pa`OeF6-(fC9M@=|D2wk6q40IFXT0+wiRBf^PA%j$PteT7w0jDWKzhQK;IJ(c;Ss zt67+9%w8wb-(*oc%CAWWP)LWOw5XRV<2Ycxaj}3qqK+m)+x(Nu3~%&(is+-`p*Yy$ zWnk-ki2mLKj3k!@bNtcDY8p{lc{^rg>ju^5ssZF>ug5d79)dsDfl&y>bV(#H7 zQ=KWVMmlkhhP&r(giSK{+@YuVn@ejg$tt+^KO8B0H3SIJJiGSoIq>UWLhei13X&|| zRP|%dq@J>7t z=kn>4wvQUTJVhEZQX%a%r6H^#|HFeN!U5gQ!-qf5So3{Gnay6%n{A?ILyLpKnmH;D za!zaRs3-1vr*l7Fy*p2HZVnJnZ1vqi!wUZWD(ZBM@qhdj@aLC1{~|&E`_2E4ajD(| zq@Yd}mGMR<00sWw%!8AcSHr%wJPp$XVEb$mf zivV6D)2k2d)(NQbHH%3N_9Ff1^^S*h^YoTPAoFy*-mfW$j)FMYmM3KNdvckTeLJVd z4@Jau7k2NOuIlMDU^8!RAcAy^#Fxw|{KofN)=Qp)QFp@*O@0FoE%21LERf=oRrRa#>#{Ep_MPd&Kd=KJz_C!bD3i{VeRTuSG<;M9L>&W-KQE=?clI5G;wt8~ zfJ}vL02|)|p5#*W*AFM4VH^OB!H3@VZQ`3kU5r@rn|)-GrG#_1w^ciP!*QVraK+wl zI*Iva8S14TZmf1ViUuw#)1d)B)UrKy)VO`^G>6q9pzv!w<+L6+J`EBKvVJ+R{-qC0 z`Xzjti)kDRC=dpJT4;lKNS8`Df(g|eJp+i-Z!nYLe<#uwB_-Jk3JP9VQsV4N$VxQt zCIaU5#I&@iv@~`gVD)sUtUQg4CYVYiFs$r}UOx^-5ZjfpOd? zkHTPlSpg+2-PbFEurN;w!{(HpIn`XLd5|>;7b4_eUw7YJI;)ynvDA`S;JJJ=9w}pu z)TH@LFTgSGZ*zV3RgTAHuADd7P%h=}sUVxW5nya}dEl@dVJb-3;lYb6Wx##pH@z0R za*>T0+wh_CU8(7n&_lbG8Kt8xz9()$=_E{14-}nc*S2Aze|w>TNr%y7bI%D%ZK27> z#2XEOFfuK|*Uvg?>00lu>`Vf}Kgv2g?U+`Si%)!qSThlj}$r*Vx#Q?XA@ zuKFxfN0EZQkT-sys69LKC0;<^liMt=Q;< zU6`s|fE`krCP_~4_LF&mih5ZWRg)eabsOhLN088lf&I&|$U?$)Yf4Ga^`L9E2|_5fYdQtmZsk?%zvK+GBSL1fK63}49I)_=I}Gq z6v}&s)|>44L2_Oa&!HW=J&6Q7lp6_Cy8tCL)X z#IOt9rMH#V53A--t&mXJZ|iy&iF_?)$`x;Aox{fW|7d&fpr+dHYglg;L_kyo>8LcN z3J6G7QBbMUdy!7)J%pklASy(92azT%NC~|o1PCPb&_fRxAoLaj-@*HNe)G*Y?>pZg z@4RP*8HO2>bFOopefC~^?X}tgKX#YkyM)bwutcf{5T5{s9lc`FQa-!6>{6zv?$YL{ zZ$2|_^3r9Bi>p$p3jAc4b#}Z>+sdkf-1FpHO=r&5eTreh`%=t@Y8cuCYr8SPHX9Hf z?UAuH`!P<E~qpBz#zn>2bgmO=v|Bl1WzFw;r=CSrNJLgnvbNmg%F@&+@(L9RX z9h%zw41V7GZI|VXsXGlV9KHK6Z{p;dT#&<)eWf{kO~gX3W~kI z;4kZP+}hf|vFtD1;Keo_diI?3o~|7`K&!w$!*Idvh2uPEoIKO2SK2qnl4TKv*!8%Q zy6NGXq(x2CM<$4Q8C=@-1j%P?{EZ@ucm9g8aAM6x{}hu!XKMICPX~XGQ}#6;-r4?B z#6u;e@PSV^X%%-rEP$qAh9^M#5AZoX@t)MT8JH8zH`HIMt^a#MfTZ^MgYEB< z1)}gwvqL+*hgq`?okyn0`1s!!O2xCk_5qA-k0m833JMAy=&A&%9L2`PeHgM6aMfU!j5NLpZ^P*Opqyf%#d{ zRxH5xCd)DL7rw=FvJmL)x4q+KI}Qpah1-Zf8NiAzX94dlm7PBT8ZSL(2*;FBQgrU8MqQhr1TW@g@65be}=F|?|fqNr$wH(dUEe;n#t=jBhrd-vu1q( z)ex7*2er@UEad<~6es?++_GRO^ol|t^#{i;oY2>;--Z_Fk0RNgIXg51&f#PhO%<9K zh(XBiS>gN+yKF`>;A~nhwO5pynZN#Gc6r)=xBx$jF$-etF17KOE5F|y4h1+Vs_b{e zFyG|?9W~42JsSl@ZyHyV$z_|_>~0VG(bXLyWAQn`5IG!Xl^+S2m-c&{6@dM{XFEXGY@zV7Uv-Hl^P9bnhNW#>U!n%_H5% z18*rD`fa!(5uCPrbZ5DiZt7sjX&eN8dRTmT}4Rr=q=y3AZ7$#uUc!ox??1Mj`=x{Dduuth9+?tn~uYGkH11^d4As4i`Fr}(=h0{J(pnMBF?Dj#MH%A=pxtTB zOR^Zm-j}XgKKQ!zC+HXTX0ElV2unKN8eGPM5(@=UtcQWxx%$pcNjlt%!=$vB;1cNr zbD*@-Zyq_al$!eqi9$vnUM`+2zf>gQ+ySU*OlzrE;aaj{r4H3pRokOviND-Q^u^N+ z)5+|FjM#%HZZ6B6sA>Hs=Zf*f_VKYhP2NNMCVBjtnp%Ae3T18^wcwD+=_UyKzSNTc znm%fu?AvlN6wn0)EYx3eo*ml!*7rYJa2XRm;_-l-MBxWE<`SnpL_5s~61mee9^uRI zi^p_N{}lpxWeUvAE@T0!_QyEk^N~hhLTcvbwi0%>{X=^h1p7=hXmX)3T%|^$`RmK} ziIL5o_UgF^X;?deD?f4_Hp{EBHXUyWxMNDRdSBao*2%Z^r@eY%9O9zr=>Lx?Ub=ty zU9XC!dwD@CV`fs4X=-dKTA3d-Dk$rFc=cO*`(3ArDoQ}M)=Ir0RsEZ4JWsN8tNvip@c$6icXpNkrXF4SC;eMPXE$A z-W9w~ZaP?y9w^5Dp61tkX>Sh;ROx2EeU{0>ktf-y+VjpeWG9JfN|lBx@_{ zlp_ALND*hbmsc|8^I)`5dxLmHC>U!9keKm5VXL44Mn#^+`iZGtO-}N?y_3LB^C+Qo z@*_bJia4V_qXq88>a}AQ7S^QJG*+%8=%r~W<295~-NeU2DkGlWG?x@xqvD6%OgdBw zooeZp%V#^-e}H^2<%qeO(Dq9>Hq9xjRS4B|PHk2>-~wnQA5eYl;@zhKgDU<@YiBe7 z$dxh((bZh4WtMX;Pd>wcT+L%7g_TIB2N=Ew8d~pu?l6g~;&yXS?^G{ zpUq_h<7W>&?kr575pPoJ)ukkqdhbW}^(j5UMfY_zpDnQa11vYk$wqP_4ZAS>tz=$XpS6yt&bH7SfVHq>acx|KcFHrX&v3-Gs? z$}QR)o|gE514i_cA)_4u`!w|PHpWGyGsu$o&1lJ9h_Bw&dExi0V9ugJc zs+LiG*(0$;5jR=dj)zV!6%*9*(>PQrj-TKTJJph{Q%alme#_dJWj5@P0fy^z!f)T! z)S6Jn3F-U;#d@=-;&V-F{B9fSC3U=#w!m|MMCW(29DQ~X&JUd!$Xr>T|~+Wv-d==c_a7?lk!%m zOjFa$D0TSa#rnz_Gsd|w^r@zz=9WY1;_;`=W}UMYS002Eh zfbNL*KF4jZqV#Z}_m})2_QP5m&`0JJWXtA3{pL|nOo5Cp8`jyhzYaOJ`iM8IbZC5t zf9R3;oj{ZQXz_r;M!xsef>D+9fe|nOD>htUNHK@nf;)Q30g6h2+NIj4GzB&@U%LnY zTmq`*#;pcSsm_&~YUA$mI6)J8#f4Mr6K$$&=4}@&j@%9-ez* zVswCEi-&r*tfOY%O&IXL3_}oVuCEK)JGT9_LApPal@9xFUF&vs@iKWh zc((+q8aw|%)=VZ#oSh`@mS-=BI^}ioI~Lf5EZ7@Q{emSYCM1NB*9Uzp=lVRKP-K`5 zB9v~P1h5Y5o*1s|`qSvC7ToIC>_K2to|YNrmYWxOWK?G>#SS#-Xq^DuyER$gs{y8K zUtjq+HA9=2KfqI~7y1x{PEu*W`!+zAOz4exyA7C4h9$gigl_@R=t)`<$jHA#^pE^Q zhs>JH_PP7y@No8#8$|@b%54P{FvR{UCynyv?;J8bc7MFHo%>}0n3WaXc`z`Jo}GPq zA{zj+XCmz&rd5BPcaAkGUB6afp&umY7Hhxs*)6e4ovYA%j6yd=nxbXx9J!l#Lz*e<_@Td@IPJCrC!^##m0Bk zFi-iIit&HJAy&`nbP}oxq_(Y#Zf#B>Iv>o~ZE)cWy#P_=zo<_pMG!hEToC2M;mxz( zqmVwOg?2ulk7`ywK~sJ2+32T_TES&_IDf9&zWAFY+fPziv;umcrn4Zq4Ow4?bMBmg z^nQv>Gcd`nO0yCZt2JQTFLW;Bvn;tj<_Z79gLdaR0tDTo??*i;L>h;E)2Yo~G-J^ExmMveYUn{;En zSFVqE6COrrc6^yXO@(=yLO3>Rn^*ze$LN?=yV>I8F7Eye{Yzz<qSC?%dfOx2cm z!|gq|O=o|c&i}UZ7X_Oh4C|PJp$%Dd4ruO3<0?wqt3^>Fzlce-w=48@&4_1D)Y{5N zv8iq|8oDvdL?!*aD&vh8TYho~6;*hRW-7Dg>`46Q!!voJCS3IgBU?G;hu&bNAb^`5 z3YNuK%AW4$|2a6X$6s8l6iRC@2$#YVkO}-YO%<{bLf?kueRpoU!_6(6TdeR&X*WRh zia0`IW_Q>X7lsHy9cDwzDesFlJmXeHHy2ttgb%B?0JRQIz+~yO_Z-B2%(X}Y zLT2sEfUe|`Sm7*_EgSS~hIt~H={>MymKy^E$Za9xt9p-{Z>ShDUfVuQ$}i*Qav9-^ zN9YKoK3RHlC6lyS#Avr1;u0j1t(M5II~GSv>^kx6dVo_n1q>Y5e1>FF?wP*uWpD-s z6)^tFMI>TVoth5z9`zbjUCT|w9Je{43@h8>IFLJ{2S@MBQh=28rIi(f9|;RRoP!2! z&b65Ju8dQWfBF4lu8>|;1)D&iIi!%6#+|f3lTSV6lB~HAqDhv=@|9a$knNmaPKOs? zz?*f>*PES*!+(!erjnyQkfdL@l#ngg7an1fEZR1|4+~a~f)s?kW;E0Eswxe_koCux z>AK~}f}vQqMn;A(I4$Fga_;QCzE5Q~e!693oJ8=V-=WlujgX0#rs{dG@TO$72^c^f zcFMmG`=!nMD`n$08pl~^jq+c&wtrS6P%}iG_g|QPH^A#ki;_FRGd6;TQRMXp&)TC* zL`+7R&5)R13BQEaW;SXMh^bx^0ds-WOtLeLA2mL>cuxl%vL}iA6cns>YDA0-G2C*y zyh85MbPZ61+`>1DK5|)Q%K4RNmV7JEz-yhCZW`n|-UP{9wfyC^K&et;E$+j%hxR;? zbIU83Q{w*aIrUbn0^`2rBr`gQ@F%?2}?#j$0Rv%v)tuSsPh&vFPIT-FEVnC%lR znxl#XQ3J5pbdnGt;pFU$xrNg!{TeyxGhdW>>Y3nT;qY9f%hBgG2cY&8ZO`5Jws2g+ z%E6GAH~wKn>ID3GNe^AGUt;l$`SYjO)wD|98Q_i-U%dD@*M5gxSd5nkW%{v93zt;B z{BuFC%I>RfBwAfa;)@tX^O1Eig^+?%^7e8ZA4tX2`|Txu{W5w$FNYRTWRig2OO+@N zDKawoxYFhNNG83^oUv0iH3XnZ=bd#F8JT_jlJ3!HjbpWE}pSD;bPg`<5B0Dp@aX4oUu{s=J^;Edi2$EGhlNm+E1zKt=RU~x(r zmJk!m^pSHE1O?uMIeA1Im~PLEPput00<&pAH97wUz#gQ+ZhLa4n%sF@-Pn($(wqR; zB;5M(!I-s_4|9Nej4&1p%Fvtf+)@kzvM_gMHRj3Y{A5Kcn)(J(vVB-k;|^;RV5`Ng zH|y1AP2My#Qt7|f;bGJSdx-X`ws*hipbPs{^YP=Z&6$ts5WTEST_~~?K+79i!l6`5 z&@%B%z3j1;3*ybPrdzSuilXN4s3px>ubB0jhg@nHJMu?io}5WXm1gdYR~~pSK-WG( z>5-4QM#j-CPd)gX#+cGF5IygqLku}?Ozk5VaKcc`yBfrjjF|*5x6*4`-g7y`(X`PL zE!P0l>7p0sS3gXI+lx(_vBlGuncr@sfJzYTn0PHJT#h69Gn9kCQj}l(ekD5-(mcG9KR( zFVGu0f_ZXm1m;$QH(2G$JBQU#fsZ!tGXL@=kb|59NJs`1_U)T9K5P|(Y4g^2-ByRuoh0GjAh zGyhSf{KFjJx%LqlyZXj3U zGI(D=V5`vCIf!cLB37uys%JWQ`JA`cxnBRsXPgP}uDV!l3YOWl7~36G!kVA8>YuB^ zlmIh8z;wf-z-9lTNNs?P-0MDWBGldt=#t4>Gy>RL>#wrd=(x2%0NAvGlBHo9_91MTR`0DRMT;)j9DU;-k(kd`B)JB;Thv#WzDaQalwOW z2s;MQDwWUgRRqM8dhcfcUecfZf}g#RBT)^q9wh=Yw|RKs>!=>NEm9eN3QqXGlXBKy zWgkM@6H9~uB02SmVfHVp1E8X41D|v4!)goi>+G$UBurO*%xw7eovSR6i*}mVesFN~ zZkM^in4G7$jmpdFf33ywkr!LEbhy#h$R<#e7m`574Yd2|bHnLEjrfMZI}_p_tQ-eU z+i4v(SRkZF)s3J;kv4lzx)QJ{kw)ntYd&W;l+KZ6$J;W;hmVU*`uJm68^gXTRzt3- z^)0d^RDSd38|#l37M;o%f_zcmXwI%h)yOA?EZ>_gLv7| z!Z6`D$eArx9N_84M=BOfFWf`_&4v z5qT7h5pj&p;bLtv&{X_lodLer2r%{A8VWB>wg#&g4|Fe$`;T~-bq0(K#l^}>copf0 z>;R1zIMZ5xVF4JmIxe7>z?=x5T!ZHd9A;8+%XQbeOrHr<&)MYtk2fmf94Gf1juWL2EOg zK(Z+OV4%s9oaJc8TSyIeA{(aV_PBwN0+o2ZRm)PNsd&JCr=kAuR!y1I{TP!B)enup zRB9gua`I7VR)X!*cI!2{@`p9@Yo$>Q2hsR>DH`?&px$JFPD($rq}6*jWa&ahC0jPi zq##L_M>Q+6rO&o8HX~!Bb@VmuP3?v&s@nFG3RM|{xN!e&rR#_vKu}MJ#od+5_hT`h z(QDy8ElVq~u|A8>@|9x{KLAE38mgt?F6$W{ETtx7?fF`#EC<03+Pt6*wAbH08E}m` zPs`xh40nYuTZtNq=fg>qeunkn`dXKf6@L#xM8K8+%YcyV3na>K9DC30J*5j(I>RD3 z1<=C7&wyD$WPUSadUwZ9aWY3BFWm=HJ){*YN8Ue_&@a#UK}TS6uis2oO%BmVgDpz5 z7Gyp$&Xc$-9s-k{Bb%BSZ~EBq$U6`BcN$62YYBWHo|H$&-4eY&J7k)RWmh9F(zWxA z#w|YRSv!MbLVWDNg$Rp~cil{6f7WhAZLIW z#pgM0c^ZbhfOjz%rg=^!IX|vk14#(quzLTyReJ2;p2GcicW1jt9eO{n6=NK<5kB$( zGjn$@3C-VSXmSR(j!yJa%vk)0fBWDMrP7W4jxdF$e+<-t9XYCIMXj2l4ok%)UYdJk zZslt&{L0$H@A{*5C)e4VrZnP}yD$^KI>fzGaFkSE-sucntK{d^>r)e5BIB(W{roL%sLY7HB5mm;WJstxiu7@h$~3QGhseHC z-8ZNocvm91PwKvj0L%Uw&7l@x37aNHD#T(AorGY6FV0UD_fL}jIWpWueDJ|Km zL9z;h-_0>E8%?cSB}xmJ?g^8fzoOIvLVjv4CM&ko4{mn)<>>uUARg^I5s zwYth_wEr?0i(QAbc152%^0tt}#=|1KHP)^83J%0+85OwJcfDqNYc`taN)Y??w!1fdgqXzxMKz?5)gP^A0`m#6nF;d+xFv2h}dab>U*GmDPwPOQl=Qvaw#) z`Nl;6GR*fYscf6cQHX{ z)Ot0EC|UKX!1QtYgJ@z;z-$vssfBkwq`P-ZY`@09M&>%3rEbXmC`G4(&?R%=ryWRO zt;O%VdH2LWn*+qCA0bw2`Jo?3lG|zHtty91VMwZ?|ZzPl{O*mOMqd2>0bXnqY4+uq>6J9uAj1JN6%?& zJNtiWY-m73<)=QR5D;(7BO5xa_F?aTt7F0t=5y`it*e}s{SZRqotqaa@811x4irAW zvQ9n+|1f{~^2)QfH!fazk;~C|Fasf6{O9AxZ^zh1(U6PJD}s;8Lm4}i*9GN z0mn3jHzhXZSAEC`$@o$gSv3%knwk7_Dz)G5@LD!((DIL%X~A#YkM(bd#+B5iyZU-V zFeZS@3x!PP;;Y93Fp$;jiJqC!jdf%0KGLfSh|}SNLBTO+3|{J6f4~f@jM^XULX1wB zVRvD3(Q&;`ARUY>iAfIh-oHdt6ZEMVF$yC|QTorLZ=#my*w}6o`~nqTDk;5obS%D1 z=I$ZL&CC0yt*wpo!Ka7&ACMF@G>J7ePqq#Y?g$A9VKj|}jssYkn0NyN1LqeO@P^ke zT-flICFAjAT;F~{OQf)Sk`tf7RJppyR#U5qJ*fPs&bH#`MI7pRTk>wCfAd|ej6KHJ z=P4XFAIw0bL$ zWBL=pjhFr6h4aG|=>aV%oF4Idmy^Sm>T@c6T3BBw>9)7Ey=R2hEjQ)$ZHOC5^{L6= zO4HKBWzihyXZc9;vb$Jl%l)Gl)uV*|v3>Kx1R(xSBsG;0$s)FiE}*cJH?_sAFrO;k;E9tXTw;s8!5 zd;4Y${O!>vWbIAtf`pj=uye=7ix**BPcl!lN=;r5s8>o-ZRf}xPmzRe=kjK&2d=DnWlt6dDJtkdX;9c)d1#GvD}d$3K);m zq$MIscp3P{C*=S055UrN`Zt7jAh2=93^gTNkOjs{zVnNjHM}AB&%al-dA04o*T?qk z@an&RY~eWn>c5BY3{Q~zT{)IXW#e@ z>J8p4%o{7;N_#?gk5bR&n8}rcsHkpah3|kdJ_Aax}S~Ng)!QA@yBqoPUXyr2G zXt{XW)LtL1zQV*%c64;S*7h81#XPiaj?KT%$sO@@MMb+3I1NFIoB3QD+?P$7)sybA zLi~B&qQIP0ZU|>^)g-3eaoPqG^Mz6t{6_ANy!LX5G0t!voS(M;hd8OvC~f)Sij$?S zU|e83Vq&`}3p2f5-#S=kYj(7Xe!acGPP-HyO`B%j{AyCrtg9yZi?Pwn@#6=qBHo4m zBz)$rIe(jf7uea>2M`O9+WvYX;#V%S)vTVYlMl@uFJkY)$wR-lg&Ben-#7>|xB%B(BE{cZ;VQkeI+%+j!f+DW#>T}u;9QmDLHKV`fHwVZZjNIQ^oD3d z&8g>V!zg1@czAsLs^B?TAZe(l<5`e;0wF9iA+0qz$2nI7%*ThjsCpLTt$cxY$SgZKlpMf7r~_59SpW!%o%sz>=G`}DvR?4gKJUq z6Lxo*=;*i#G;`erUQhVrPQ(dQd`@hI5H;88v(2}R@{qDAquk=fJ~sX!)pR~(j8j5K zkwh0wkc#KN5l`|zL$8;=@6wlFXOMiRd34l0xMyV5R{T=4;i)sF(8yRarB$#;arw(@ zuk;V?8O6O*s9d9jod(`e`tB@Ut-A0QD!H9GUT)XN%?KcY@KUmDg|v0C%NbSXp*mU1 zsD$yhNh}ZQ+k(Yhc>ww%<0FS3Yg0ZN&!boqL+&B0+R{2=q(L^X@tG4Si&_s;K=k$H z=Vdh4L#@Mc5xNBy9J*kKQ@QhLZM8ci<@#>4l=ZcI>Zq=-FN@Yr$)6x3*H|E%D-XAa ze!dp*=;ozN*}?TX9Psi=O~AH@a|??E`LpF-Pi!N0 zndMA7pBz}DKiufXS2kBgcw1t6)Yq#@FpTs!@4eh!K0#-8Og^jhINO!x0&g!?FYN`6 zx%?z?gaw+dW6YH?u#!aG_<|0#z8fUtv?wwoltql8Yp#7tqi}OIwSUJqozJq6geWZS$J!V{a^cAqHb4}yR zHM4x5f!Dj2HGi11QKtU*VVV1S`p>pMl04TctHIq2?moDZ!CUS;DJbc>n1p{&9i@|6 zrF=jL*MFolP@CXXPovL;n^90I#&5?RA+0eV6zG#kQZ|K-6J?nFA@l9(Ezw0Q{f{qR zDE2_GR55S0aE1^v+pCrDy^|{B1O-jK3G(~_VUcC~eVL-E@U47#>S~Pj=f%XkxUojE z$jWc(^)+atCbiKgxZJGQsM)W&9}J40I#ax1zZ;7afD7}*lW{XSY)CS=i{FsV)b-=e zn-ug6jJUSkW22M6Ll$Y zpbhlMsQTT4IJgPoLLtk@J@V#GY&MTt4<&m5kMus^8y=B&In1?fjCiF?Bfmw2e zV`QXdffI}SZjGncgg+7%Al|g}z4^3ETs2V~9{1pzQl5~pMJc@gIJ!S}-_+8UA2`X1 zSI3q1S3f&nVrn?>3WQ$oWpakzVRPc#oAB{2X?#r0hQ?HbfY}9ic{Z4lYPmJ)Edc3h&7MD#H1nlkm^^^Yv^-#stta{Zik6H7WS_aHAYDT@oWc9z-- zCwm6D9@ml3 z$|cl2_U2L32{F(E7x|%{+S5nlkVn4@=>>*&w#&^0%>b{x_XVsF8Yw4lm6XvUXS$#w z38z?0h8+OujJ}$2Bz}_7Y)-fmG66GuTd3p5Iw|9bN8awXo$eRY@L&5NwcTvX9?xFi zEExZWzKV8mqk}1m4xMS#L(g~j$lWGB+(9o|icVQMx}nx;D@9fJL*YmcQ`l&!?hG#z z8n=KKu(aWiv#kq!sNO_MuJm8vSlxW)!o08p=Sh8?GF)J_KRDw3)h7caz-lP_zy-wd2v)uY1@bwZ)eW-`*dW!frGruP5)^c5cd%7&LlFa6pH4nOntDyB)>i=ng_*5FQnDbz zK;wdzZfGr`|1BlMmpi>2(`JFsGd!Y33QQ=syeGUSGcM`gJU-aHLhtV5j2YY?+53WV zto&^3bhdl(bo=MfXsO)H@N z`+}>9-YATDKusUSBK%BzV~j8Ik2bv-y9?f(2-^t{EIX-PbevAZM|eZ08$eLf09)F_ zSG>1{tnO)7H-&J`2SPcrvmv zI1i_>UOBWcCv4t?tUz*=@j|QV;XSvLX4-zw(b~M(Lzuy6IQIVvImv>5;=-w z<=MP@c)8)XhULAUzcOh#V!2LQbE2$-GlQe}8PR0R_UTS1T6w?r2dc0Ol?Zu{#Mh~) z%q|X8?e3aMz{lhsElzbo#vwBUACoDj$KQp_ir?dPf8U*=HF+1RY|?Hsc3I5<~T*QD!Zs|Ptq{4cXtuPq%bQKiDv(JhoaaS z-lEOtl;Tzl{^wRFm-fcp?H|TPAe{&H6+UR=@3fiq?mKTeLjjkcc`DSFE>ecV0WQ-@>>kNIzS47&>6 zjcis)ZtmqJeqKh}FKBVUtoOfz&QwkrDH2o{t5x@F-uaVR;^KN&F1ce{ZaM zI!UU^`5hu?VJJ`}aR2_pIG+55Q!7z>fZ{b2hw`dPc~)DhTpvz6^fk(LeCL<;sY{gp zK8RR5`N++;7+D#kS*nGev&I=$wr$l53ccQQ=R&dVlpiowQ?U}YmbzlP1M61heb=P@ zhxh(GS613B>)AjKTzgF|L93_xx0-Ry_6-eJnu?3NFD2E(txZ8i(P9lZHcsZy${5|$Aa~aR^#D7 zb-5+>+3V#6zRbn}(`!rJ_lEA2#Q8KG8_Wc)QrK%W0xssZV|F-}QDkCzW}+lgck;yp zrc--{8Ygt%>h|7K>5WTp-tql(2+0nyzV*h4@*d@4k4-2gwi^vyg~oO-VQ2jf9fE{}=w%RC{|hph5{IMgK%miF>fx`7Aj z%|TPM)cS7m@B_W6ssUlP#Nu)OOLBfDa%bS2G#NTUv$N8WlA#{7Zn|GrjU0Q=FNz`S zzP$A9gKC1&SkXO;p(yrO3y?znSI3Ukr8(-DD9vogHaB_J&3Z4KsYnC#3RZ^=@{-Im z$Yi5Hdm>nq`sJNsL!l5q&zN4ue@bL4k-i9VRqLUs>2jj^r(J(|NzUu|K?9$sfI`pv zp|HIi|FYoiDRMLA5zUuN>$XPYkUoxPK z;o+0=mk)_V9!3XjVWqOC5zfb@=e;_*KxLT1PEkG6<=#TBv13qmr;t}g_h(O*uCwRO zr3wlP9cZ+=e;VPu0v@s4#ERJNaTmPZZaY&aciq`z5iy*x9~#@cR`ZQ0-GHRcNKIOa zD)k>0tYANTqB80@^RN*cxb>rTLDJaGvCO}g)1cf+d^97~dvYZCSA{LgPRE9%^tIx% z1a)JE(^iWYC0nWD_>U_~-JN?mwr5x%BodIcXGCo_-=;6cQ@?a(g0dp7K;IP6i}8C6 z{dq#YQHiXO!?cLFiOmG)Ad%&8BTkon{us=Nb&bR7kHPPX;MIYkpADz>^5Cs@7wfJX z`_Y$D9vV^*=%=m+lw)yKv8c6^qn2o323)?2Idd=88eiQ-0``>3fK>EXou^WhWO=HV zI=N=2B0V;LMz<)7;vaHn@u-hWQOgS{qD`>UWhvImV^p))Gr3Og(t(XJIA2^$wIAtj zy4=|_DIgs`O_D)$st>a?0X@arZI2#U{;KjHS`ON*O{P8^^9NiCNA?EmKEGj$1?+f!f>fBid&vhG@z+( z!sM};pb@*@{a)^v0`2loalC1mgaXmG!lSH^jxeW~DSP>nYb9%r*-lm7`0ECB-bGVp z6~R{5R2Y0TY{@dq?Ba{DRmeIPJ=s->pie%?=&l)yAQ`n#thgWK+-^JD(&;pu{K;^$qgEVL*E13{%-*8 z_U2miBE9pUCtsX_R5j5+SWZ}bTsj2BdCt8zI>vdB08@D7GZaEP4G}hk$j@GLm zMGZgX(PD0Q#hUcoM%mKtuk`*&4*AEr19R>^xq6h;c7jw&ywL* za3H$1!H%5W>viRL)1N3JoK>bGk3%=GV`!ufziHw2+j?VAO-Y5xnI3=@p63Oy4pw%# z!2O-802EPs$hAYKC!C=Zy4pKhDs%cPXIDAnSAQ1m6T79cFHkwps^GVmYOp+gF_Ov{ z{T4mAr!e@>9MSQ9Osi#H&sE!W?*VTpIv*kU_EB};i?xg^L_-tRL*M)X?_ml@Zj`~h z3T@Dx^{H`@`q6*x)TOs0wgpV%IuEqr!mp-d3=`s%L!nIOe`=z$n~5#(#fh8$Jwp@W z<%RV>&ez$YVHt6Ha;tnm!dF$sW+Yy`Zg`1v-TONGG~MTo3vY|y#8dI*&%E?PbaH=u zp@szkwHJ2Y{0Gg0b(n}wia86dUOHK-P(ybmHL<_hAv)A0{Nt0i-^VnOqYRWxRf1zo zG7d>Ad2=~}&4z}uENRg}vws1uDH(0KnYNAP9;I1_X;eJ4`qF>UT1@c&0IUDEfXDwO znz zTyzySd|h%eS$BEh<>B!V0pH)RQN+KuwdEyn<@`Hm2N`z$|KQjE zA{d$T6y>N102LU`=tgpAS7j*D8atOABdF?7U;z7g+iMg80oPX5c)xx7_7*)o_p@iu zU|bgjxn*RGfgAQ4i#2P2wG*&?n-VBd zrBeGH$ND~Ch$Z1O%XL}7Od6JZ3pN6`TwH`GCFPIoYk=4O2)rY*3_u9$GsbG3U}7%w zw6$4pxKo>`Pv9e^y^)2glj}#hFA%@G$^m$i(0T(CgB+?Fu1W_YluBB~i{%o19&&}K z+EG(OeczA#UK`L1!d@OCt=d1GMb3K)&iNsOQ0|p#eX`ae0z~3unV@79Hx{s@mB zvOOhH8M_d#vzCcRMamvz%x#PwR12TuStg}v7+{HPt>@D^U|e?wTLxKWGSm--NYky^ z$G;Y|=w1I!&N+Yv#==COG*_;fJGjD$61*Ofw>^hcf8!dqi(kDXPXeV@bTMklko@CABzqMYV64>BIou`AQHT z?=m_#G6?=+pBe@gRSjX*Q^dX{8cIYzJ}oGIv#~j#ftO2{0Y9&9vqrI# zkfSc~itwUWGiwH@0zv*bzHt8&2crVbx}U#0I=fH5S?YG@5wWif3m6#bmi+cgYYu%S z*ZF-icd`$#9IS?NR%js{&>;EUVPXJ1p`P?MfZGAs{30I3x|mvR_6;f}x1Od|c?F!s z({60Uty*+rkCetdZ|F|4W44=2Gqd;^Ux=>q)i`b-miv>MuWMGj+f%s8T{3H)eQ zI|Jw2at2I1hLx1cug?c@OD(58V)m&9B(x^c^ATZQ8sj2Yx<9llnE)mEbTJx>D6xqA z93$Wn=FtiEC!{_brDi^lG4GjM^OX|?3Fkq^&8>`H>ObuT&D_n{$^xx))=SQ#F(V@W zgZ$v{x?B%|YCp$q)ZTh=_a)yF((LpbkVU;NBH~}|iF62jA0&a|g&u-E$dGHfMY;N? z#+sZiYSOypR!voB(F;6FL@c5kcwEQu`Kcp`-#zV}QqP_v^oQKowQ}Dq8OA(jt#HS~ zNP|`O0l2g?>Wb%4>EGN$@4RW`wdQi!H^(vQb#<~ra;#b#=wGQB)jTIV-GlZTm%jl( z!`Pr@fmrQ_B+f@pOv#TRWvn2ec-+PegGWuIl{LQS&^+S)mG@VXYMwA{DIxE2Gwg83pQQYA)FMhp*6u88TTwIETghe#f)7j zxzVviCznmeUB#Wuwj7d;E5sb)lL|&7wb3sL_F*j->Pvxi-e3<$ptU(i=S%L3J+b_r zqZU6{3(MA>)31N``r$XeRQy-k9JS=O;Q7ka0;{D?r*bc}9C&FfkK!WSgW8WYv1EF{ zF5~x^6|%5D9V2Z-KdEQ$Luy5V?Im;@$x3E=Z>GIHNk33qnP}`Vkbl`JbzrfV#~LFX z@IF9%)Onf<@c6uGW@dI|x$T%N5>yL>vNr&{lkUA)R%pEbkDTX;Cy;H0dX>k&W;W4C^Za-rCv%AXy(wqYe~RY+vGl{C6fR7FQe?;lS`v$nK! z&2z%=(t2^Sh`tXA9mwW<@8@wqo64%Zf7+qsdbF%E8FPI_=8j=~FtFED0#7|+R6TQO zIac-aOie#NtClWtZAelsslQFS-cMoUSwm6sZ8$utv@o{+JZxNW<(Q(22n~B>*o~<; zdsJjN-1D94!>Tc;gSXQmj_;n3%<;E|@uplB872oE+X$1wOYY0xJqMRt>$W#MpyO_p z6Rv0-mC^W%M)mhG1^0DS6momErQD4Tp#+TEqk^l{Ft zhDhZSX64B|YW}Wm_fst?kSkE9Sz^D#Otr2NW(om^Gq+g1{5TlJrwoYh~{QyK5oh{Y}nHFsI7jJeCC;6 zBewV#`a=YI4izG1K~(H}(sLF#AH90B`pDCb5Tx3>e;?63cLWbT(~3$`<8N;3pFs86 zKvLnOK;5vzKfXKYJeumg0mtxK)?FwXmJV2MyVktpH0jZ@?}PqY(wgCYFycy3RO9@C zN#gKOTpc-*JgP!|**{!Pzq1sXpJITqGzm&b^u|S1yf!VAT+h-8FSN>%85MNf`hTiB z�!-Eeww`uofcYXb31=aBv1esz?=3M=60Qu|PrzM5%#GBd21H5(5(z^I zEkQyG5LAjZkuVApI)j3NKZ)EGxy*9f7V%NecyNXdiVQ2=WNM=$S6G0$V((o z)8Cp~UV%mW*$CF+7sg&t7B=bSj_L45d9ddYn~bO^a!ui2!pAg^X_4&Ydoj-<0@E2s zKp@q9#db)q+z<2~Cp~dz1uN3Sv^Yo-wAx*~OoRC$R6H16+2#pU$5E0;@F`d09GYD} z8Pzg9$X>0B6)Hup^-A`Is9CUQhBbyZf2gQfNFZc;SN<_&;1$weyB=deYiA;i!?suC zcCiXk)YjOkJ_fcNFa~z~x#6P#_JJAEVYP{>M#!VS>AcEaod<*( zDe64}4bT0>FZ?iyyq(dEhn{}-TXKqZ*uct~xI@=1A98RnKM2p~- zD1o{x<&Ro5y{XFh#buzWCY|fsG_(%6JdtJA?j6vxZSZ3kyDRZL&3&+XhXTKX`B)I@ zA10zQ)$O?28t1}3u~`_~s0--o#f`zXK&DhQ2_C%>VcHV3*+tQ+>Z=+D`tm{rI71b96H zGj*jy8-5hIZ?#=*R57p7Us!5frXi;Z`J*4#^-OX=(bY|l;HE-jLO;PF?(|UL`w{|U zXD+3bF4GBFYY`h}Ff)W^&!ha50Qe4W5s3dL2rXX-l<(g5P1=zi1+{vkD*utcQg# zM4HqYm|B{m;z1dEmt}LJ)1v()yxsZ7BCwCiJ_3i6tcm279~7sSFwNNRJM)6%;TOMl#k`B*&Eh|K z?49|)Y{Iwv_{YS=@?iyMZqTmio^q(A*sEP{RVt7gPX&Bz|H1kLyoKnqmHk(GoZSis zxLgJ9`nn3S0Uzw>L)AN0D#JTNt}V2vH+yOXku65CtMs|$352T==UgWJ`0};B;UlUF z{K^B?Mv8n})u+V{EYeu}VLwnB>0t5;Z1(|6TCYn*5VvfY>FTYG^7U}-38lNDLwZi@ z#?s3WF;r&cb#wszCT&n$504EOiYezB4v&$%u%5gHg9Sl1-cZ{;uOFU&k&tL;$3`3w zS{Mz4&2oT)ks$J3#`Knr?)I-h9^5z0hl;%cSRZItRH;oN1$QR2J~(^c86DZP2unW` zPk8W52ZG492bh!)uSWLJnb>iYY!-4hd~w6TpN*w?(_C}YfCOnMiJW%da1alR&*b^g zKpl_Q&}>`TO7}*??B0_P7BhYOC>d~`5I@r!WoataJhFL7x|VnS=Luga&w|BQCJFfx zgZ(^PAtL=HFNR#)%uug_dzpbK9_MBCd9!#EemVADnLHRt z#eAc(9*B@3@?(>_2Cdz7!pQ0mbJw|(7$c~nobG%z_D+GaYtyIrUrzr-@Tc;fj00iW zE>>a=qKz_T%tL@;YNv0@Qe@wychv02w&x%jHL5kg%h}{;6Vh<(u3xovV_fNu7o}RF zqig*WykovvZ#ysC61|ljD}+v>m1#+WB6x~PVl4)o`KBG23419S3jj1#m7|w5zmb=9 zEK01_UOoTryZ;rj{Kp>O?yAiLM4n^q8`Ifok?mK!?mSFvGR|{bm^$ z2MO8y=5oKaJ@_{XI@qRGyr;AP+wouVpa1yb{tZ<6e|^(CtvmZ(vp{)QBXAwNJ8xqN Kv!GwPb??9Rmsg(v literal 0 HcmV?d00001 From 2d671cddfe0266183b9e694eadb6305ff95824bd Mon Sep 17 00:00:00 2001 From: frayle-ons <194791647+frayle-ons@users.noreply.github.com> Date: Wed, 24 Jun 2026 12:09:21 +0100 Subject: [PATCH 3/8] render image in eval notebook --- DEMO/evaluation_workflow_demo.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEMO/evaluation_workflow_demo.ipynb b/DEMO/evaluation_workflow_demo.ipynb index ee6d67d..7d72f02 100644 --- a/DEMO/evaluation_workflow_demo.ipynb +++ b/DEMO/evaluation_workflow_demo.ipynb @@ -29,7 +29,7 @@ "\n", "Currently the evaluation module only evaluates single label predictions meaning that, while ClassifAI is designed to return a ranked list of several semantically similar candidate entries to a provided query sample, only the top result will be considered when comparing the VectorStore result to a ground truth label provided by a user.\n", "\n", - "[IMAGE HERE OF THE TOP ANSWER BEING COMPARED AND THE OTHER ANSWERS BEING DIREGARDED]\n", + "![top_1_eval_image](files/eval_top_1_diagram.png)\n", "\n", "The Evaluation module is currently in development, and in the future its feature set may be extended to include a broader range of evaluation tasks such as multi-class multi-label classification, where potentially multiple labels for a ground truth sample can be compared and evaluated against mutiple ranked candidate predictions of the VectorStore." ] From f8bb435f3c271addc521b6e9f764332ca2337b38 Mon Sep 17 00:00:00 2001 From: frayle-ons <194791647+frayle-ons@users.noreply.github.com> Date: Wed, 24 Jun 2026 12:12:07 +0100 Subject: [PATCH 4/8] added content for eval module to the main readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2600c4a..6f95193 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,8 @@ Further guides and tutorials can be found in the [DEMO folder](./DEMO/) of this - [General workflow](./DEMO/general_workflow_demo.ipynb) - A general introduction to using the ClassifAI package. +- [Evaluation Module](./DEMO/evaluation_workflow_demo.ipynb) + - Evaluate and compare VectorStore performance in a multi-class single-label setting using your own ground truth data. - [Custom vectorisers](./DEMO/custom_vectoriser.ipynb) - make your own custom vectoriser model that will interact with the core features of the package. - [Custom pre- and post-processing "hooks"](./DEMO/using_hooks.ipynb) From 8b0d7ae32603d5009aa58b728cf7b658b2e0dc83 Mon Sep 17 00:00:00 2001 From: Jamie Milsom Date: Wed, 24 Jun 2026 14:36:20 +0100 Subject: [PATCH 5/8] fix typos Co-authored-by: Jamie Milsom --- DEMO/evaluation_workflow_demo.ipynb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/DEMO/evaluation_workflow_demo.ipynb b/DEMO/evaluation_workflow_demo.ipynb index 7d72f02..55ced2c 100644 --- a/DEMO/evaluation_workflow_demo.ipynb +++ b/DEMO/evaluation_workflow_demo.ipynb @@ -13,12 +13,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The Evaluation module provies a toolkit to evaluate the performance of VectorStores in a mulit-class, single-label classification setting. Provided the user has:\n", + "The Evaluation module provides a toolkit to evaluate the performance of VectorStores in a multi-class, single-label classification setting. Provided the user has:\n", "\n", "- a constructed VectorStore (or multiple VectorStores) built from historically labeled (or similar) data;\n", "- a held out collection of labelled ground truth data, not in the VectorStore;\n", "\n", - "then this module can be used to evaluate VectorStore peformance, presenting results with a variety of available metrics that can be specified by the user." + "then this module can be used to evaluate VectorStore performance, presenting results with a variety of available metrics that can be specified by the user." ] }, { @@ -31,14 +31,14 @@ "\n", "![top_1_eval_image](files/eval_top_1_diagram.png)\n", "\n", - "The Evaluation module is currently in development, and in the future its feature set may be extended to include a broader range of evaluation tasks such as multi-class multi-label classification, where potentially multiple labels for a ground truth sample can be compared and evaluated against mutiple ranked candidate predictions of the VectorStore." + "The Evaluation module is currently in development, and in the future its feature set may be extended to include a broader range of evaluation tasks such as multi-class multi-label classification, where potentially multiple labels for a ground truth sample can be compared and evaluated against multiple ranked candidate predictions of the VectorStore." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "For multi-class, single-label evaluation we have implemented several metrics that can be calculated, there names and descriptions are as follows: \n", + "For multi-class, single-label evaluation we have implemented several metrics that can be calculated, their names and descriptions are as follows: \n", "| Metric | Description |\n", "|------------------|-------------------------------------------------------------------------------------------------|\n", "| Accuracy | The proportion of correctly predicted labels out of the total number of predictions. |\n", @@ -63,7 +63,7 @@ "\n", "- Setting up an Evaluation task using the fake queries file and a list of metrics to evaluate.\n", "\n", - "- Typing this all together, evaluating the VectorStores against the ground truth query file using the evaluation module.\n", + "- Tying this all together, evaluating the VectorStores against the ground truth query file using the evaluation module.\n", "\n", "\n", "See the ClassifAI GitHub repository and `DEMO/README.md` for information on accessing the associated demo datasets needed for this (and other) notebook tutorials.
\n", @@ -81,7 +81,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To begin, we're going to create 2 VectorStores from our `fake_soc_daraset.csv` file which contains mock SOC survey responses and their corresponding occupation codes. One VectorStore will be built from the full dataset, and the second one will be built from half the dataset. \n", + "To begin, we're going to create 2 VectorStores from our `fake_soc_dataset.csv` file which contains mock SOC survey responses and their corresponding occupation codes. One VectorStore will be built from the full dataset, and the second one will be built from half the dataset. \n", "\n", "Since the second VectorStore will contain only have the training data, we can reason that this lack of coverage will showcase poorer performance against a evaluation dataset that assesses the full coverage of the training data." ] @@ -153,7 +153,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If the code cells in this section ran succesfully then we now have 2 VectorStores we can use to evaluate with the Evaluation module." + "If the code cells in this section ran successfully then we now have 2 VectorStores we can use to evaluate with the Evaluation module." ] }, { @@ -170,7 +170,7 @@ "ClassifAI provides an `Evaluation` class (in the Evaluation module) which can be used to load a collection of ground truth queries and specify metrics to use. The evaluator constructor accepts 4 arguments:\n", "\n", "- A pandas dataframe with `['text', 'label']` columns/headers both of type string, which represent the text sample queries and gold standard ground truth label respectively,\n", - "- A list of evaluation metric names which must be strings correpsonding to one of the current avaialble metrics: `['accuracy', 'macro_recall', 'macro_precision', 'macro_f1']`,\n", + "- A list of evaluation metric names which must be strings corresponding to one of the current available metrics: `['accuracy', 'macro_recall', 'macro_precision', 'macro_f1']`,\n", "- A `batch_size` which determines how many samples should be processed at once (smaller size will take longer but be more memory efficient),\n", "- A boolean argument `save_output` which determines if generated results should be saved to CSV.\n", "\n", @@ -278,7 +278,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We've already demonstrated the core featrures of the Evaluation module. This final section will show some additional implemeted features including:\n", + "We've already demonstrated the core features of the Evaluation module. This final section will show some additional implemented features including:\n", "\n", "- Efficient ways to load VectorStores to avoid memory issues when using many and/or large VectorStores,\n", "- More of the available metrics,\n", @@ -352,7 +352,7 @@ "source": [ "## Thats It!\n", "\n", - "The results (this time with more metrics) from the last evaluation run should be saved to the file in the current directory as `evaluation_results.csv`, since we didn't specify a file name this time to save the resutls to. Remember you can disable writing results to file by setting `save_output=False` in the Evaluation constructor.\n", + "The results (this time with more metrics) from the last evaluation run should be saved to the file in the current directory as `evaluation_results.csv`, since we didn't specify a file name this time to save the results to. Remember you can disable writing results to file by setting `save_output=False` in the Evaluation constructor.\n", "\n", "Finally, one more reminder that this module is still in development and may be subject to breaking changes in the future.\n" ] From 8b4c24b39a1832630acf0e668dea16a62252670d Mon Sep 17 00:00:00 2001 From: Erlend Frayling <194791647+frayle-ons@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:08:43 +0100 Subject: [PATCH 6/8] Apply suggestions from code review Co-authored-by: Jamie Milsom --- DEMO/README.md | 2 +- DEMO/evaluation_workflow_demo.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DEMO/README.md b/DEMO/README.md index 948069e..915d583 100644 --- a/DEMO/README.md +++ b/DEMO/README.md @@ -84,7 +84,7 @@ It covers: * Memory-efficient evaluation using callable functions to load VectorStores on-demand, useful when evaluating many or large VectorStores. -**Note:** The Evaluation module is currently in development and its API is subject to change in future releases. +> **Note:** The Evaluation module is currently in development and its API is subject to change in future releases. --- ## Installation of classifai diff --git a/DEMO/evaluation_workflow_demo.ipynb b/DEMO/evaluation_workflow_demo.ipynb index 55ced2c..5e5da1f 100644 --- a/DEMO/evaluation_workflow_demo.ipynb +++ b/DEMO/evaluation_workflow_demo.ipynb @@ -240,7 +240,7 @@ "results = evaluator.evaluate(\n", " vectorstores=[demo_vectorstore_full, demo_vectorstore_half],\n", " vectorstore_names=[\"full data vectorstore\", \"half data vectorstore\"],\n", - " output_file=\"./classifai_temp/demo_eval_results.csv\", # leaving this line blank will save the results to\n", + " output_file=\"./classifai_temp/demo_eval_results.csv\", # leaving this line blank will save the results to evaluation_results.csv\n", ")" ] }, From 7e2d5ab5c93567f165a56698ee49f7b3abceac52 Mon Sep 17 00:00:00 2001 From: frayle-ons <194791647+frayle-ons@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:41:02 +0100 Subject: [PATCH 7/8] set all vectorstores to skip save in eval notebook --- DEMO/evaluation_workflow_demo.ipynb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DEMO/evaluation_workflow_demo.ipynb b/DEMO/evaluation_workflow_demo.ipynb index 5e5da1f..9311f25 100644 --- a/DEMO/evaluation_workflow_demo.ipynb +++ b/DEMO/evaluation_workflow_demo.ipynb @@ -103,7 +103,7 @@ " file_name=\"data/fake_soc_dataset.csv\",\n", " data_type=\"csv\",\n", " vectoriser=demo_vectoriser,\n", - " output_dir=\"./classifai_temp/full_vectorStore/\",\n", + " skip_save=True,\n", ")" ] }, @@ -145,7 +145,7 @@ " file_name=\"data/fake_soc_dataset_half.csv\",\n", " data_type=\"csv\",\n", " vectoriser=demo_vectoriser,\n", - " output_dir=\"./classifai_temp/half_vectorStore/\",\n", + " skip_save=True,\n", ")" ] }, @@ -308,8 +308,7 @@ " file_name=\"data/fake_soc_dataset.csv\",\n", " data_type=\"csv\",\n", " vectoriser=efficient_vectoriser,\n", - " output_dir=\"./classifai_temp/efficient_vectorStore/\",\n", - " overwrite=True,\n", + " skip_save=True,\n", " )\n", "\n", " return efficient_vectorstore # these functions must return only the VectorStore object." From af04908e672fe9d6a7c8d13475f752e3c1cab8be Mon Sep 17 00:00:00 2001 From: frayle-ons <194791647+frayle-ons@users.noreply.github.com> Date: Fri, 26 Jun 2026 17:46:18 +0100 Subject: [PATCH 8/8] notebook cleanup and adding metric_results attribute to evaluation class --- DEMO/evaluation_workflow_demo.ipynb | 11 ++--------- src/classifai/evaluation/main.py | 7 +++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/DEMO/evaluation_workflow_demo.ipynb b/DEMO/evaluation_workflow_demo.ipynb index 9311f25..43dd1f1 100644 --- a/DEMO/evaluation_workflow_demo.ipynb +++ b/DEMO/evaluation_workflow_demo.ipynb @@ -83,7 +83,7 @@ "source": [ "To begin, we're going to create 2 VectorStores from our `fake_soc_dataset.csv` file which contains mock SOC survey responses and their corresponding occupation codes. One VectorStore will be built from the full dataset, and the second one will be built from half the dataset. \n", "\n", - "Since the second VectorStore will contain only have the training data, we can reason that this lack of coverage will showcase poorer performance against a evaluation dataset that assesses the full coverage of the training data." + "Since the second VectorStore will contain only half the training data, we can reason that it will not perform as well as the VectorStore built with the full dataset due to lack of coverage." ] }, { @@ -149,13 +149,6 @@ ")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the code cells in this section ran successfully then we now have 2 VectorStores we can use to evaluate with the Evaluation module." - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -248,7 +241,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The results object is a dataframe with provided VectorStore names as the row indexes, and each column is associated with a give metric. We should also see the results have been saved to a CSV file in the specified directory." + "The results object is a dataframe with provided VectorStore names as the row indexes, and each column is associated with a given metric. We should also see the results have been saved to the specified output CSV file." ] }, { diff --git a/src/classifai/evaluation/main.py b/src/classifai/evaluation/main.py index abdb374..100ec8b 100644 --- a/src/classifai/evaluation/main.py +++ b/src/classifai/evaluation/main.py @@ -131,7 +131,6 @@ class Evaluation: batch_size (int): Batch size for vectorstore search operations. save_output (bool): Whether to save evaluation results to a file. parsed_metrics (dict): Dictionary of parsed metrics to compute. - results (pd.DataFrame | None): DataFrame containing overall evaluation results. metric_results (dict): Dictionary of individual metric results for detailed inspection. """ @@ -163,6 +162,7 @@ def __init__( self.ground_truths["qid"] = self.ground_truths.index.astype(str) self.batch_size = batch_size self.save_output = save_output + self.metric_results = {} # parse the provided metrics and store them in the instance try: @@ -290,11 +290,10 @@ def evaluate( # noqa: C901, PLR0912 del resolved_vs # Compute metrics for the current VectorStore and store results - vs_metrics = {} try: for _metric_name, metric in self.parsed_metrics.items(): result = metric.evaluate(results_df) - vs_metrics[result.name] = result.value + self.metric_results[result.name] = result.value except Exception as e: raise EvaluationError( "Metric computation failed.", @@ -302,7 +301,7 @@ def evaluate( # noqa: C901, PLR0912 ) from e # Append the current VectorStore's metrics to the overall results DataFrame - vectorstore_df = pd.DataFrame([vs_metrics], index=[name]) + vectorstore_df = pd.DataFrame([self.metric_results], index=[name]) overall_results_df = pd.concat([overall_results_df, vectorstore_df]) # Save results to CSV if requested