From f9ec6b72475152ed147f8336bdf3cab83a390543 Mon Sep 17 00:00:00 2001
From: Bagatur <22008038+baskaryan@users.noreply.github.com>
Date: Thu, 19 Dec 2024 08:51:35 -0800
Subject: [PATCH] Update agent eval tutorial (#582)
Co-authored-by: Tanushree <87711021+tanushree-sharma@users.noreply.github.com>
---
docs/evaluation/tutorials/agents.mdx | 2226 ++++++++++++++---
docs/evaluation/tutorials/index.mdx | 2 +-
.../tutorials/static/agent_tutorial_graph.png | Bin 0 -> 37935 bytes
.../tutorials/static/chinook-diagram.png | Bin 0 -> 208720 bytes
docs/evaluation/tutorials/static/qa_graph.png | Bin 0 -> 8904 bytes
.../tutorials/static/refund_graph.png | Bin 0 -> 13445 bytes
.../tutorials/static/sql_agent_graph.png | Bin 0 -> 12720 bytes
src/css/custom.css | 75 +-
src/theme/CodeBlock/index.js | 172 +-
9 files changed, 2053 insertions(+), 422 deletions(-)
create mode 100644 docs/evaluation/tutorials/static/agent_tutorial_graph.png
create mode 100644 docs/evaluation/tutorials/static/chinook-diagram.png
create mode 100644 docs/evaluation/tutorials/static/qa_graph.png
create mode 100644 docs/evaluation/tutorials/static/refund_graph.png
create mode 100644 docs/evaluation/tutorials/static/sql_agent_graph.png
diff --git a/docs/evaluation/tutorials/agents.mdx b/docs/evaluation/tutorials/agents.mdx
index c66aacdb..09e0b543 100644
--- a/docs/evaluation/tutorials/agents.mdx
+++ b/docs/evaluation/tutorials/agents.mdx
@@ -1,370 +1,867 @@
----
-sidebar_position: 6
----
-
import { RegionalUrl } from "@site/src/components/RegionalUrls";
# Evaluate an agent
-In this tutorial, we will walk through 3 evaluation strategies LLM agents, building on the conceptual points shared in our [evaluation guide](../concepts#agents).
+:::info Key concepts
+[Agent evaluation](../concepts#agents) | [Evaluators](../concepts#evaluators) | [LLM-as-judge evaluators](../concepts#llm-as-judge)
+:::
+
+In this tutorial, we'll build a customer support bot that helps users navigate a digital music store. Then, we'll go through the three most effective types of evaluations to run on chat bots:
-- `Final Response`: Evaluate the agent's final response.
-- `Single step`: Evaluate any agent step in isolation (e.g., whether it selects the appropriate tool).
-- `Trajectory`: Evaluate whether the agent took the expected path (e.g., of tool calls) to arrive at the final answer.
+- [Final response](../concepts#evaluating-an-agents-final-response): Evaluate the agent's final response.
+- [Trajectory](../concepts#evaluating-an-agents-trajectory): Evaluate whether the agent took the expected path (e.g., of tool calls) to arrive at the final answer.
+- [Single step](../concepts#evaluating-a-single-step-of-an-agent): Evaluate any agent step in isolation (e.g., whether it selects the appropriate first tool for a given ).
-First, we will build an agent using [LangGraph](https://github.com/langchain-ai/langgraph).
+We'll build our agent using [LangGraph](https://github.com/langchain-ai/langgraph), but the techniques and LangSmith functionality shown here are framework-agnostic.
-## Set up environment
+## Setup
-We'll set up our environment variables for OpenAI and and install
+### Configure the environment
+
+Let's install the required dependencies:
```bash
-%pip install --upgrade --quiet langchain langsmith langchain-community langchain-experimental langgraph
+pip install -U langgraph langchain langchain-community langchain-openai
```
+Let's set up environment variables for OpenAI and :
+
```python
+#region
import getpass
import os
-def _set_env(var: str):
+def _set_env(var: str) -> None:
if not os.environ.get(var):
- os.environ[var] = getpass.getpass(f"{var}: ")
+ os.environ[var] = getpass.getpass(f"Set {var}: ")
-_set_env("OPENAI_API_KEY")
os.environ["LANGCHAIN_TRACING_V2"] = "true"
-os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com" # Update appropriately for self-hosted installations or the EU region
_set_env("LANGCHAIN_API_KEY")
+_set_env("OPENAI_API_KEY")
+#endregion
```
-## Configure the database
+### Download the database
-We will be creating a SQLite database for this tutorial. SQLite is a lightweight database that is easy to set up and use. We will be loading the `chinook` database, which is a sample database that represents a digital media store.
+We will create a SQLite database for this tutorial. SQLite is a lightweight database that is easy to set up and use.
+We will load the `chinook` database, which is a sample database that represents a digital media store.
Find more information about the database [here](https://www.sqlitetutorial.net/sqlite-sample-database/).
-For convenience, we have hosted the database (`Chinook.db`) on a public GCS bucket.
+For convenience, we have hosted the database in a public GCS bucket:
```python
+#region
import requests
+
url = "https://storage.googleapis.com/benchmarks-artifacts/chinook/Chinook.db"
response = requests.get(url)
if response.status_code == 200:
# Open a local file in binary write mode
- with open("Chinook.db", "wb") as file:
+ with open("chinook.db", "wb") as file:
# Write the content of the response (the file) to the local file
file.write(response.content)
print("File downloaded and saved as Chinook.db")
else:
print(f"Failed to download the file. Status code: {response.status_code}")
+#endregion
```
-We will use a handy SQL database wrapper available in the `langchain_community` package to interact with the database. The wrapper provides a simple interface to execute SQL queries and fetch results. We will also use the `langchain_openai` package to interact with the OpenAI API for language models later in the tutorial.
+Here's a sample of the data in the db:
```python
-from langchain_community.utilities import SQLDatabase
+#region [collapsed]
+import sqlite3
+
+conn = sqlite3.connect("chinook.db")
+cursor = conn.cursor()
-db = SQLDatabase.from_uri("sqlite:///Chinook.db")
-print(db.dialect)
-print(db.get_usable_table_names())
-db.run("SELECT * FROM Artist LIMIT 10;")
+# Fetch all results
+cursor.execute("SELECT * FROM Artist LIMIT 10;").fetchall()
+#endregion
```
-## SQL Agent
+```console
+[(1, 'AC/DC'), (2, 'Accept'), (3, 'Aerosmith'), (4, 'Alanis Morissette'), (5, 'Alice In Chains'), (6, 'Antônio Carlos Jobim'), (7, 'Apocalyptica'), (8, 'Audioslave'), (9, 'BackBeat'), (10, 'Billy Cobham')]
+```
-We'll use a [LangGraph agent](https://www.langchain.com/agents) with access to a set of tools for working with SQL:
+And here's the database schema (image from https://github.com/lerocha/chinook-database):
-
+
-### LLM
+### Define the customer support agent
-```python
-from langchain_openai import ChatOpenAI
+We'll create a [LangGraph](https://langchain-ai.github.io/langgraph/) agent with limited access to our database. For demo purposes, our agent will support two basic types of requests:
-llm=ChatOpenAI(model="gpt-4o",temperature=0)
-experiment_prefix="sql-agent-gpt4o"
-metadata = "Chinook, gpt-4o base-case-agent"
-```
+- Lookup: The customer can look up song titles, artist names, and albums based on other identifying information. For example: "What songs do you have by Jimi Hendrix?"
+- Refund: The customer can request a refund on their past purchases. For example: "My name is Claude Shannon and I'd like a refund on a purchase I made last week, could you help me?"
+
+For simplicity in this demo, we'll implement refunds by deleting the corresponding database records. We'll skip implementing user authentication and other production security measures.
+
+The agent's logic will be structured as two separate subgraphs (one for lookups and one for refunds), with a parent graph that routes requests to the appropriate subgraph.
-### Tools
+#### Refund agent
-We'll use [SQL toolkit](https://python.langchain.com/v0.2/docs/tutorials/sql_qa/#agents) as well as some custom tools to check the query before executing it and check the query result from the database to confirm it is not empty or irrelevant to the question.
+Let's build the refund processing agent. This agent needs to:
+1. Find the customer's purchase records in the database
+2. Delete the relevant Invoice and InvoiceLine records to process the refund
+
+We'll create two SQL helper functions:
+1. A function to execute the refund by deleting records
+2. A function to look up a customer's purchase history
+
+To make testing easier, we'll add a "mock" mode to these functions. When mock mode is enabled, the functions will simulate database operations without actually modifying any data.
```python
-import json
+import sqlite3
-from langchain_community.agent_toolkits import SQLDatabaseToolkit
-
-from langgraph.checkpoint.sqlite import SqliteSaver
-from langgraph.graph import END, MessageGraph
-from langgraph.prebuilt.tool_node import ToolNode
-from langchain_core.messages import AIMessage
-from langchain_core.prompts import ChatPromptTemplate
-from langchain.agents import tool
-
-# SQL toolkit
-toolkit = SQLDatabaseToolkit(db=db, llm=llm)
-tools = toolkit.get_tools()
-
-# Query checking
-query_check_system = """You are a SQL expert with a strong attention to detail.
-Double check the SQLite query for common mistakes, including:
-- Using NOT IN with NULL values
-- Using UNION when UNION ALL should have been used
-- Using BETWEEN for exclusive ranges
-- Data type mismatch in predicates
-- Properly quoting identifiers
-- Using the correct number of arguments for functions
-- Casting to the correct data type
-- Using the proper columns for joins
-
-If there are any of the above mistakes, rewrite the query. If there are no mistakes, just reproduce the original query.
-
-Execute the correct query with the appropriate tool."""
-query_check_prompt = ChatPromptTemplate.from_messages([("system", query_check_system),("user", "{query}")])
-query_check = query_check_prompt | llm
+#region [collapsed]
+def _refund(invoice_id: int | None, invoice_line_ids: list[int] | None, mock: bool = False) -> float:
+ """Given an Invoice ID and/or Invoice Line IDs, delete the relevant Invoice/InvoiceLine records in the Chinook DB.
-@tool
-def check_query_tool(query: str) -> str:
- """
- Use this tool to double check if your query is correct before executing it.
+ Args:
+ invoice_id: The Invoice to delete.
+ invoice_line_ids: The Invoice Lines to delete.
+ mock: If True, do not actually delete the specified Invoice/Invoice Lines. Used for testing purposes.
+
+ Returns:
+ float: The total dollar amount that was deleted (or mock deleted).
"""
- return query_check.invoke({"query": query}).content
-# Query result checking
-query_result_check_system = """You are grading the result of a SQL query from a DB.
-- Check that the result is not empty.
-- If it is empty, instruct the system to re-try!"""
-query_result_check_prompt = ChatPromptTemplate.from_messages([("system", query_result_check_system),("user", "{query_result}")])
-query_result_check = query_result_check_prompt | llm
+ if invoice_id is None and invoice_line_ids is None:
+ return 0.0
-@tool
-def check_result(query_result: str) -> str:
+ # Connect to the Chinook database
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ total_refund = 0.0
+
+ try:
+ # If invoice_id is provided, delete entire invoice and its lines
+ if invoice_id is not None:
+ # First get the total amount for the invoice
+ cursor.execute(
+ """
+ SELECT Total
+ FROM Invoice
+ WHERE InvoiceId = ?
+ """,
+ (invoice_id,),
+ )
+
+ result = cursor.fetchone()
+ if result:
+ total_refund += result[0]
+
+ # Delete invoice lines first (due to foreign key constraints)
+ if not mock:
+ cursor.execute(
+ """
+ DELETE FROM InvoiceLine
+ WHERE InvoiceId = ?
+ """,
+ (invoice_id,),
+ )
+
+ # Then delete the invoice
+ cursor.execute(
+ """
+ DELETE FROM Invoice
+ WHERE InvoiceId = ?
+ """,
+ (invoice_id,),
+ )
+
+ # If specific invoice lines are provided
+ if invoice_line_ids is not None:
+ # Get the total amount for the specified invoice lines
+ placeholders = ",".join(["?" for _ in invoice_line_ids])
+ cursor.execute(
+ f"""
+ SELECT SUM(UnitPrice * Quantity)
+ FROM InvoiceLine
+ WHERE InvoiceLineId IN ({placeholders})
+ """,
+ invoice_line_ids,
+ )
+
+ result = cursor.fetchone()
+ if result and result[0]:
+ total_refund += result[0]
+
+ if not mock:
+ # Delete the specified invoice lines
+ cursor.execute(
+ f"""
+ DELETE FROM InvoiceLine
+ WHERE InvoiceLineId IN ({placeholders})
+ """,
+ invoice_line_ids,
+ )
+
+ # Commit the changes
+ conn.commit()
+
+ except sqlite3.Error as e:
+ # Roll back in case of error
+ conn.rollback()
+ raise e
+
+ finally:
+ # Close the connection
+ conn.close()
+
+ return float(total_refund)
+
+#endregion
+
+#region [collapsed]
+def _lookup(
+ customer_first_name: str,
+ customer_last_name: str,
+ customer_phone: str,
+ track_name: str | None,
+ album_title: str | None,
+ artist_name: str | None,
+ purchase_date_iso_8601: str | None,
+) -> list[dict]:
+ """Find all of the Invoice Line IDs in the Chinook DB for the given filters.
+
+ Returns:
+ a list of dictionaries that contain keys: {
+ 'invoice_line_id',
+ 'track_name',
+ 'artist_name',
+ 'purchase_date',
+ 'quantity_purchased',
+ 'price_per_unit'
+ }
"""
- Use this tool to check the query result from the database to confirm it is not empty and is relevant.
+
+ # Connect to the database
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ # Base query joining all necessary tables
+ query = """
+ SELECT
+ il.InvoiceLineId,
+ t.Name as track_name,
+ art.Name as artist_name,
+ i.InvoiceDate as purchase_date,
+ il.Quantity as quantity_purchased,
+ il.UnitPrice as price_per_unit
+ FROM InvoiceLine il
+ JOIN Invoice i ON il.InvoiceId = i.InvoiceId
+ JOIN Customer c ON i.CustomerId = c.CustomerId
+ JOIN Track t ON il.TrackId = t.TrackId
+ JOIN Album alb ON t.AlbumId = alb.AlbumId
+ JOIN Artist art ON alb.ArtistId = art.ArtistId
+ WHERE c.FirstName = ?
+ AND c.LastName = ?
+ AND c.Phone = ?
"""
- return query_result_check.invoke({"query_result": query_result}).content
-tools.append(check_query_tool)
-tools.append(check_result)
+ # Parameters for the query
+ params = [customer_first_name, customer_last_name, customer_phone]
+
+ # Add optional filters
+ if track_name:
+ query += " AND t.Name = ?"
+ params.append(track_name)
+
+ if album_title:
+ query += " AND alb.Title = ?"
+ params.append(album_title)
+
+ if artist_name:
+ query += " AND art.Name = ?"
+ params.append(artist_name)
+
+ if purchase_date_iso_8601:
+ query += " AND date(i.InvoiceDate) = date(?)"
+ params.append(purchase_date_iso_8601)
+
+ # Execute query
+ cursor.execute(query, params)
+
+ # Fetch results
+ results = cursor.fetchall()
+
+ # Convert results to list of dictionaries
+ output = []
+ for row in results:
+ output.append(
+ {
+ "invoice_line_id": row[0],
+ "track_name": row[1],
+ "artist_name": row[2],
+ "purchase_date": row[3],
+ "quantity_purchased": row[4],
+ "price_per_unit": row[5],
+ }
+ )
+
+ # Close connection
+ conn.close()
+
+ return output
+#endregion
```
-### State
+Now let's define our graph. We'll use a simple architecture with three main paths:
-```python
-from typing import Annotated
-from typing_extensions import TypedDict
+1. Extract customer and purchase information from the conversation
+2. Route the request to one of three paths:
+ - Refund path: If we have sufficient purchase details (Invoice ID or Invoice Line IDs) to process a refund
+ - Lookup path: If we have enough customer information (name and phone) to search their purchase history
+ - Response path: If we need more information, respond to the user requesting the specific details needed
+
+The graph's state will track:
+- The conversation history (messages between user and agent)
+- All customer and purchase information extracted from the conversation
+- The next message to send to the user (followup text)
+
+````python
+#region
+from typing import Literal
+import json
+
+from langchain.chat_models import init_chat_model
+from langchain_core.runnables import RunnableConfig
+from langgraph.graph import END, StateGraph
from langgraph.graph.message import AnyMessage, add_messages
+from langgraph.types import Command, interrupt
+from tabulate import tabulate
+from typing_extensions import Annotated, TypedDict
+#endregion
+# Graph state.
+#region
class State(TypedDict):
+ """Agent state."""
messages: Annotated[list[AnyMessage], add_messages]
-```
+ followup: str | None
+
+ invoice_id: int | None
+ invoice_line_ids: list[int] | None
+ customer_first_name: str | None
+ customer_last_name: str | None
+ customer_phone: str | None
+ track_name: str | None
+ album_title: str | None
+ artist_name: str | None
+ purchase_date_iso_8601: str | None
+#endregion
+
+# Instructions for extracting the user/purchase info from the conversation.
+#region
+gather_info_instructions = """You are managing an online music store that sells song tracks. \
+Customers can buy multiple tracks at a time and these purchases are recorded in a database as \
+an Invoice per purchase and an associated set of Invoice Lines for each purchased track.
+
+Your task is to help customers who would like a refund for one or more of the tracks they've \
+purchased. In order for you to be able refund them, the customer must specify the Invoice ID \
+to get a refund on all the tracks they bought in a single transaction, or one or more Invoice \
+Line IDs if they would like refunds on individual tracks.
+
+Often a user will not know the specific Invoice ID(s) or Invoice Line ID(s) for which they \
+would like a refund. In this case you can help them look up their invoices by asking them to \
+specify:
+- Required: Their first name, last name, and phone number.
+- Optionally: The track name, artist name, album name, or purchase date.
+
+If the customer has not specified the required information (either Invoice/Invoice Line IDs \
+or first name, last name, phone) then please ask them to specify it."""
+#endregion
+
+# Extraction schema, mirrors the graph state.
+#region
+class PurchaseInformation(TypedDict):
+ """All of the known information about the invoice / invoice lines the customer would like refunded. Do not make up values, leave fields as null if you don't know their value."""
+
+ invoice_id: int | None
+ invoice_line_ids: list[int] | None
+ customer_first_name: str | None
+ customer_last_name: str | None
+ customer_phone: str | None
+ track_name: str | None
+ album_title: str | None
+ artist_name: str | None
+ purchase_date_iso_8601: str | None
+ followup: Annotated[
+ str | None,
+ ...,
+ "If the user hasn't enough identifying information, please tell them what the required information is and ask them to specify it.",
+ ]
+#endregion
+
+# Model for performing extraction.
+info_llm = init_chat_model("gpt-4o-mini").with_structured_output(
+ PurchaseInformation, method="json_schema", include_raw=True
+)
-### SQL Assistant
+# Graph node for extracting user info and routing to lookup/refund/END.
+#region
+async def gather_info(state: State) -> Command[Literal["lookup", "refund", END]]:
+ info = await info_llm.ainvoke(
+ [
+ {"role": "system", "content": gather_info_instructions},
+ *state["messages"],
+ ]
+ )
+ parsed = info["parsed"]
+ if any(parsed[k] for k in ("invoice_id", "invoice_line_ids")):
+ goto = "refund"
+ elif all(
+ parsed[k]
+ for k in ("customer_first_name", "customer_last_name", "customer_phone")
+ ):
+ goto = "lookup"
+ else:
+ goto = END
+ update = {"messages": [info["raw"]], **parsed}
+ return Command(update=update, goto=goto)
+#endregion
+
+# Graph node for executing the refund.
+# Note that here we inspect the runtime config for an "env" variable.
+# If "env" is set to "test", then we don't actually delete any rows from our database.
+# This will become important when we're running our evaluations.
+#region
+def refund(state: State, config: RunnableConfig) -> dict:
+ # Whether to mock the deletion. True if the configurable var 'env' is set to 'test'.
+ mock = config.get("configurable", {}).get("env", "prod") == "test"
+ refunded = _refund(
+ invoice_id=state["invoice_id"], invoice_line_ids=state["invoice_line_ids"], mock=mock
+ )
+ response = f"You have been refunded a total of: ${refunded:.2f}. Is there anything else I can help with?"
+ return {
+ "messages": [{"role": "assistant", "content": response}],
+ "followup": response,
+ }
+#endregion
+
+# Graph node for looking up the users purchases
+#region
+def lookup(state: State) -> dict:
+ args = (
+ state[k]
+ for k in (
+ "customer_first_name",
+ "customer_last_name",
+ "customer_phone",
+ "track_name",
+ "album_title",
+ "artist_name",
+ "purchase_date_iso_8601",
+ )
+ )
+ results = _lookup(*args)
+ if not results:
+ response = "We did not find any purchases associated with the information you've provided. Are you sure you've entered all of your information correctly?"
+ followup = response
+ else:
+ response = f"Which of the following purchases would you like to be refunded for?\n\n```json{json.dumps(results, indent=2)}\n```"
+ followup = f"Which of the following purchases would you like to be refunded for?\n\n{tabulate(results, headers='keys')}"
+ return {
+ "messages": [{"role": "assistant", "content": response}],
+ "followup": followup,
+ "invoice_line_ids": [res["invoice_line_id"] for res in results],
+ }
+#endregion
-Use [prompt based roughly on what is shown here](https://python.langchain.com/v0.2/docs/tutorials/sql_qa/#agents).
+# Building our graph
+graph_builder = StateGraph(State)
-```python
-from langchain_core.runnables import Runnable, RunnableConfig
-
-# Assistant
-class Assistant:
-
- def __init__(self, runnable: Runnable):
- self.runnable = runnable
-
- def __call__(self, state: State, config: RunnableConfig):
- while True:
- # Append to state
- state = {**state}
- # Invoke the tool-calling LLM
- result = self.runnable.invoke(state)
- # If it is a tool call -> response is valid
- # If it has meaningful text -> response is valid
- # Otherwise, we re-prompt it b/c response is not meaningful
- if not result.tool_calls and (
- not result.content
- or isinstance(result.content, list)
- and not result.content[0].get("text")
- ):
- messages = state["messages"] + [("user", "Respond with a real output.")]
- state = {**state, "messages": messages}
- else:
- break
- return {"messages": result}
-
-# Assistant runnable
-query_gen_system = """
-ROLE:
-You are an agent designed to interact with a SQL database. You have access to tools for interacting with the database.
-GOAL:
-Given an input question, create a syntactically correct SQLite query to run, then look at the results of the query and return the answer.
-INSTRUCTIONS:
-- Only use the below tools for the following operations.
-- Only use the information returned by the below tools to construct your final answer.
-- To start you should ALWAYS look at the tables in the database to see what you can query. Do NOT skip this step.
-- Then you should query the schema of the most relevant tables.
-- Write your query based upon the schema of the tables. You MUST double check your query before executing it.
-- Unless the user specifies a specific number of examples they wish to obtain, always limit your query to at most 5 results.
-- You can order the results by a relevant column to return the most interesting examples in the database.
-- Never query for all the columns from a specific table, only ask for the relevant columns given the question.
-- If you get an error while executing a query, rewrite the query and try again.
-- If the query returns a result, use check_result tool to check the query result.
-- If the query result result is empty, think about the table schema, rewrite the query, and try again.
-- DO NOT make any DML statements (INSERT, UPDATE, DELETE, DROP etc.) to the database."""
-
-query_gen_prompt = ChatPromptTemplate.from_messages([("system", query_gen_system),("placeholder", "{messages}")])
-assistant_runnable = query_gen_prompt | llm.bind_tools(tools)
-```
+graph_builder.add_node(gather_info)
+graph_builder.add_node(refund)
+graph_builder.add_node(lookup)
-### Graph Utilities
+graph_builder.set_entry_point("gather_info")
+graph_builder.add_edge("lookup", END)
+graph_builder.add_edge("refund", END)
-We will define a few utility functions to help us with the agent implementation. Specifically, we will wrap a ToolNode with a fallback to handle errors and surface them to the agent.
+refund_graph = graph_builder.compile()
+````
-```python
-from langchain_core.messages import ToolMessage
-from langchain_core.runnables import RunnableLambda
+We can visualize our refund graph:
-def create_tool_node_with_fallback(tools: list) -> dict:
- return ToolNode(tools).with_fallbacks(
- [RunnableLambda(handle_tool_error)], exception_key="error"
- )
+```python
+# Assumes you're in an interactive Python environment
+#region [collapsed]
+from IPython.display import Image, display
-def _print_event(event: dict, _printed: set, max_length=1500):
- current_state = event.get("dialog_state")
- if current_state:
- print(f"Currently in: ", current_state[-1])
- message = event.get("messages")
- if message:
- if isinstance(message, list):
- message = message[-1]
- if message.id not in _printed:
- msg_repr = message.pretty_repr(html=True)
- if len(msg_repr) > max_length:
- msg_repr = msg_repr[:max_length] + " ... (truncated)"
- print(msg_repr)
- _printed.add(message.id)
-
-def handle_tool_error(state) -> dict:
- error = state.get("error")
- tool_calls = state["messages"][-1].tool_calls
- return {
- "messages": [
- ToolMessage(
- content=f"Error: {repr(error)}\n please fix your mistakes.",
- tool_call_id=tc["id"],
- )
- for tc in tool_calls
- ]
- }
+display(Image(refund_graph.get_graph(xray=True).draw_mermaid_png()))
+#endregion
```
-### Graph
+
-We will then define the workflow for the agent.
+#### Lookup agent
+
+For the lookup (i.e. question-answering) agent, we'll use a simple ReACT architecture and give the agent tools for looking up track names, artist names, and album names based on various filters. For example, you can look up albums by a particular artist, artists who released songs with a specific name, etc.
```python
-from langgraph.checkpoint.sqlite import SqliteSaver
-from langgraph.graph import END, StateGraph
-from langgraph.prebuilt import ToolNode, tools_condition
-
-# Graph
-builder = StateGraph(State)
-
-# Define nodes: these do the work
-builder.add_node("assistant", Assistant(assistant_runnable))
-builder.add_node("tools", create_tool_node_with_fallback(tools))
-
-# Define edges: these determine how the control flow moves
-builder.set_entry_point("assistant")
-builder.add_conditional_edges(
- "assistant",
- # If the latest message (result) from assistant is a tool call -> tools_condition routes to tools
- # If the latest message (result) from assistant is a not a tool call -> tools_condition routes to END
- tools_condition,
- # "tools" calls one of our tools. END causes the graph to terminate (and respond to the user)
- {"tools": "tools", END: END},
-)
-builder.add_edge("tools", "assistant")
+#region
+from langchain.embeddings import init_embeddings
+from langchain_core.tools import tool
+from langchain_core.vectorstores import InMemoryVectorStore
+from langgraph.prebuilt import create_react_agent
+#endregion
+
+#region
+# Our SQL queries will only work if we filter on the exact string values that are in the DB.
+# To ensure this, we'll create vectorstore indexes for all of the artists, tracks and albums
+# ahead of time and use those to disambiguate the user input. E.g. if a user searches for
+# songs by "prince" and our DB records the artist as "Prince", ideally when we query our
+# artist vectorstore for "prince" we'll get back the value "Prince", which we can then
+# use in our SQL queries.
+#endregion
+#region [collapsed]
+def index_fields() -> tuple[InMemoryVectorStore, InMemoryVectorStore, InMemoryVectorStore]:
+ """Create an index for all artists, an index for all albums, and an index for all songs."""
+ try:
+ # Connect to the chinook database
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ # Fetch all results
+ tracks = cursor.execute("SELECT Name FROM Track").fetchall()
+ artists = cursor.execute("SELECT Name FROM Artist").fetchall()
+ albums = cursor.execute("SELECT Title FROM Album").fetchall()
+ finally:
+ # Close the connection
+ if conn:
+ conn.close()
+
+ embeddings = init_embeddings("openai:text-embedding-3-small")
+
+ track_store = InMemoryVectorStore(embeddings)
+ artist_store = InMemoryVectorStore(embeddings)
+ album_store = InMemoryVectorStore(embeddings)
+
+ track_store.add_texts([t[0] for t in tracks])
+ artist_store.add_texts([a[0] for a in artists])
+ album_store.add_texts([a[0] for a in albums])
+ return track_store, artist_store, album_store
+#endregion
+
+track_store, artist_store, album_store = index_fields()
+
+# Agent tools
+@tool
+#region [collapsed]
+def lookup_track(
+ track_name: str | None = None,
+ album_title: str | None = None,
+ artist_name: str | None = None,
+) -> list[dict]:
+ """Lookup a track in Chinook DB based on identifying information about.
+
+ Returns:
+ a list of dictionaries per matching track that contain keys {'track_name', 'artist_name', 'album_name'}
+ """
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ query = """
+ SELECT DISTINCT t.Name as track_name, ar.Name as artist_name, al.Title as album_name
+ FROM Track t
+ JOIN Album al ON t.AlbumId = al.AlbumId
+ JOIN Artist ar ON al.ArtistId = ar.ArtistId
+ WHERE 1=1
+ """
+ params = []
+
+ if track_name:
+ track_name = track_store.similarity_search(track_name, k=1)[0].page_content
+ query += " AND t.Name LIKE ?"
+ params.append(f"%{track_name}%")
+ if album_title:
+ album_title = album_store.similarity_search(album_title, k=1)[0].page_content
+ query += " AND al.Title LIKE ?"
+ params.append(f"%{album_title}%")
+ if artist_name:
+ artist_name = artist_store.similarity_search(artist_name, k=1)[0].page_content
+ query += " AND ar.Name LIKE ?"
+ params.append(f"%{artist_name}%")
+
+ cursor.execute(query, params)
+ results = cursor.fetchall()
+
+ tracks = [
+ {"track_name": row[0], "artist_name": row[1], "album_name": row[2]}
+ for row in results
+ ]
+
+ conn.close()
+ return tracks
+#endregion
-# The checkpointer lets the graph persist its state
-memory = SqliteSaver.from_conn_string(":memory:")
-graph = builder.compile(checkpointer=memory)
-```
+@tool
+#region [collapsed]
+def lookup_album(
+ track_name: str | None = None,
+ album_title: str | None = None,
+ artist_name: str | None = None,
+) -> list[dict]:
+ """Lookup an album in Chinook DB based on identifying information about.
+
+ Returns:
+ a list of dictionaries per matching album that contain keys {'album_name', 'artist_name'}
+ """
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ query = """
+ SELECT DISTINCT al.Title as album_name, ar.Name as artist_name
+ FROM Album al
+ JOIN Artist ar ON al.ArtistId = ar.ArtistId
+ LEFT JOIN Track t ON t.AlbumId = al.AlbumId
+ WHERE 1=1
+ """
+ params = []
-### Test
+ if track_name:
+ query += " AND t.Name LIKE ?"
+ params.append(f"%{track_name}%")
+ if album_title:
+ query += " AND al.Title LIKE ?"
+ params.append(f"%{album_title}%")
+ if artist_name:
+ query += " AND ar.Name LIKE ?"
+ params.append(f"%{artist_name}%")
-```python
-questions = ["Which country's customers spent the most? And how much did they spend?",
- "How many albums does the artist Led Zeppelin have?",
- "What was the most purchased track of 2017?",
- "Which sales agent made the most in sales in 2009?"]
+ cursor.execute(query, params)
+ results = cursor.fetchall()
+
+ albums = [{"album_name": row[0], "artist_name": row[1]} for row in results]
+
+ conn.close()
+ return albums
+#endregion
+
+@tool
+#region [collapsed]
+def lookup_artist(
+ track_name: str | None = None,
+ album_title: str | None = None,
+ artist_name: str | None = None,
+) -> list[str]:
+ """Lookup an album in Chinook DB based on identifying information about.
+
+ Returns:
+ a list of matching artist names
+ """
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ query = """
+ SELECT DISTINCT ar.Name as artist_name
+ FROM Artist ar
+ LEFT JOIN Album al ON al.ArtistId = ar.ArtistId
+ LEFT JOIN Track t ON t.AlbumId = al.AlbumId
+ WHERE 1=1
+ """
+ params = []
+
+ if track_name:
+ query += " AND t.Name LIKE ?"
+ params.append(f"%{track_name}%")
+ if album_title:
+ query += " AND al.Title LIKE ?"
+ params.append(f"%{album_title}%")
+ if artist_name:
+ query += " AND ar.Name LIKE ?"
+ params.append(f"%{artist_name}%")
+
+ cursor.execute(query, params)
+ results = cursor.fetchall()
+
+ artists = [row[0] for row in results]
+
+ conn.close()
+ return artists
+#endregion
+
+# Agent model
+qa_llm = init_chat_model("claude-3-5-sonnet-latest")
+# The prebuilt ReACT agent only expects State to have a 'messages' key, so the
+# state we defined for the refund agent can also be passed to our lookup agent.
+qa_graph = create_react_agent(qa_llm, [lookup_track, lookup_artist, lookup_album])
```
```python
-## Invoke
-import uuid
-_printed = set()
-thread_id = str(uuid.uuid4())
-
-config = {
- "configurable": {
- # Checkpoints are accessed by thread_id
- "thread_id": thread_id,
- }
-}
+display(Image(qa_graph.get_graph(xray=True).draw_mermaid_png()))
+```
-msg = {"messages": ("user", questions[0])}
-messages = graph.invoke(msg,config)
-messages['messages'][-1].content
+
-## Stream
-_printed = set()
-thread_id = str(uuid.uuid4())
+#### Parent agent
-config = {
- "configurable": {
- # Checkpoints are accessed by thread_id
- "thread_id": thread_id,
- }
-}
+Now let's define a parent agent that combines our two task-specific agents.
+The only job of the parent agent is to route to one of the sub-agents by classifying the user's current intent, and to compile the output into a followup message.
-events = graph.stream(
- {"messages": ("user", questions[0])}, config, stream_mode="values"
+```python
+# Schema for routing user intent.
+# We'll use structured outputs to enforce that the model returns only
+# the desired output.
+#region
+class UserIntent(TypedDict):
+ """The user's current intent in the conversation"""
+
+ intent: Literal["refund", "question_answering"]
+#endregion
+
+# Routing model with structured output
+router_llm = init_chat_model("gpt-4o-mini").with_structured_output(
+ UserIntent, method="json_schema", strict=True
)
-for event in events:
- _print_event(event, _printed)
+
+# Instructions for routing.
+#region
+route_instructions = """You are managing an online music store that sells song tracks. \
+You can help customers in two types of ways: (1) answering general questions about \
+tracks sold at your store, (2) helping them get a refund on a purhcase they made at your store.
+
+Based on the following conversation, determine if the user is currently seeking general \
+information about song tracks or if they are trying to refund a specific purchase.
+
+Return 'refund' if they are trying to get a refund and 'question_answering' if they are \
+asking a general music question. Do NOT return anything else. Do NOT try to respond to \
+the user.
+"""
+#endregion
+
+# Node for routing.
+#region
+async def intent_classifier(
+ state: State,
+) -> Command[Literal["refund_agent", "question_answering_agent"]]:
+ response = router_llm.invoke(
+ [{"role": "system", "content": route_instructions}, *state["messages"]]
+ )
+ return Command(goto=response["intent"] + "_agent")
+#endregion
+
+# Node for making sure the 'followup' key is set before our agent run completes.
+#region
+def compile_followup(state: State) -> dict:
+ """Set the followup to be the last message if it hasn't explicitly been set."""
+ if not state.get("followup"):
+ return {"followup": state["messages"][-1].content}
+ return {}
+#endregion
+
+# Agent definition
+graph_builder = StateGraph(State)
+graph_builder.add_node(intent_classifier)
+# Since all of our subagents have compatible state,
+# we can add them as nodes directly.
+graph_builder.add_node("refund_agent", refund_graph)
+graph_builder.add_node("question_answering_agent", qa_graph)
+graph_builder.add_node(compile_followup)
+
+graph_builder.set_entry_point("intent_classifier")
+graph_builder.add_edge("refund_agent", "compile_followup")
+graph_builder.add_edge("question_answering_agent", "compile_followup")
+graph_builder.add_edge("compile_followup", END)
+
+graph = graph_builder.compile()
```
-## Eval
+We can visualize our compiled parent graph including all of its subgraphs:
-Agent evaluation can focus on at least 3 things:
+```python
+display(Image(graph.get_graph().draw_mermaid_png()))
+```
+
+
+
+#### Try it out
-- `Response`: The inputs are a prompt and an optional list of tools. The output is the final agent response.
-- `Single step`: As before, the inputs are a prompt and an optional list of tools. The output the tool call.
-- `Trajectory`: As before, the inputs are a prompt and an optional list of tools. The output is the list of tool calls
+Let's give our custom support agent a whirl!
-
+```python
+#region
+state = await graph.ainvoke(
+ {"messages": [{"role": "user", "content": "what james brown songs do you have"}]}
+)
+print(state["followup"])
+#endregion
+```
-:::tip
+```console
+#region [collapsed]
+I found 20 James Brown songs in the database, all from the album "Sex Machine". Here they are:
+
+1. Please Please Please
+2. Think
+3. Night Train
+4. Out Of Sight
+5. Papa's Got A Brand New Bag Pt.1
+6. I Got You (I Feel Good)
+7. It's A Man's Man's Man's World
+8. Cold Sweat
+9. Say It Loud, I'm Black And I'm Proud Pt.1
+10. Get Up (I Feel Like Being A) Sex Machine
+11. Hey America
+12. Make It Funky Pt.1
+13. I'm A Greedy Man Pt.1
+14. Get On The Good Foot
+15. Get Up Offa That Thing
+16. It's Too Funky In Here
+17. Living In America
+18. I'm Real
+19. Hot Pants Pt.1
+20. Soul Power (Live)
+
+This includes many of his most famous hits like "I Got You (I Feel Good)", "It's A Man's Man's Man's World", and "Living In America". All these tracks are collected on the album "Sex Machine".
+#endregion
+```
-See our [evaluation guide](../concepts#agents) for more details on Agent evaluation.
+```python
+#region
+state = await graph.ainvoke({"messages": [
+ {
+ "role": "user",
+ "content": "my name is Aaron Mitchell and my number is +1 (204) 452-6452. I bought some songs by Led Zeppelin that i'd like refunded",
+ }
+]})
+print(state["followup"])
+#endregion
+```
-:::
+```console
+#region [collapsed]
+Which of the following purchases would you like to be refunded for?
-### Response evaluation
+ invoice_line_id track_name artist_name purchase_date quantity_purchased price_per_unit
+----------------- -------------------------------- ------------- ------------------- -------------------- ----------------
+ 267 How Many More Times Led Zeppelin 2009-08-06 00:00:00 1 0.99
+ 268 What Is And What Should Never Be Led Zeppelin 2009-08-06 00:00:00 1 0.99
+#endregion
+```
-We can evaluate how well an agent does overall on a task. This basically involves treating the agent as a black box and just evaluating whether it gets the job done or not.
+## Evaluations
-:::tip
+Now that we've got a testable version of our agent, let's run some evaluations.
+Agent evaluation can focus on at least 3 things:
-See the full overview of agent response evaluation in our [conceptual guide](../concepts#evaluating-an-agents-final-response).
+- [Final response](../concepts#evaluating-an-agents-final-response): The inputs are a prompt and an optional list of tools. The output is the final agent response.
+- [Trajectory](../concepts#evaluating-an-agents-trajectory): As before, the inputs are a prompt and an optional list of tools. The output is the list of tool calls
+- [Single step](../concepts#evaluating-a-single-step-of-an-agent): As before, the inputs are a prompt and an optional list of tools. The output is the tool call.
-:::
+Let's run each type of evaluation:
-`Dataset`
+### Final response evaluator
-First, create a dataset that evaluates end-to-end performance of the agent. We can take some questions related to the Chinook database from [here](https://github.com/brianchiang-tw/SQL_for_DataScience/blob/master/Module3_Practice_Quiz).
+First, let's create a [dataset](../concepts#datasets) that evaluates end-to-end performance of the agent.
+For simplicity we'll use the same dataset for final response and trajectory evaluation, so we'll add both ground-truth responses and trajectories for each example question.
+We'll cover the trajectories in the next section.
```python
from langsmith import Client
@@ -372,231 +869,1130 @@ from langsmith import Client
client = Client()
# Create a dataset
+#region
examples = [
- ("Which country's customers spent the most? And how much did they spend?", "The country whose customers spent the most is the USA, with a total expenditure of $523.06"),
- ("What was the most purchased track of 2013?", "The most purchased track of 2013 was Hot Girl."),
- ("How many albums does the artist Led Zeppelin have?","Led Zeppelin has 14 albums"),
- ("What is the total price for the album “Big Ones”?","The total price for the album 'Big Ones' is 14.85"),
- ("Which sales agent made the most in sales in 2009?", "Steve Johnson made the most sales in 2009"),
+ {
+ "question": "How many songs do you have by James Brown",
+ "response": "We have 20 songs by James Brown",
+ "trajectory": ["question_answering_agent", "lookup_track"]
+ },
+ {
+ "question": "My name is Aaron Mitchell and I'd like a refund.",
+ "response": "I need some more information to help you with the refund. Please specify your phone number, the invoice ID, or the line item IDs for the purchase you'd like refunded.",
+ "trajectory": ["refund_agent"],
+ },
+ {
+ "question": "My name is Aaron Mitchell and I'd like a refund on my Led Zeppelin purchases. My number is +1 (204) 452-6452",
+ "response": 'Which of the following purchases would you like to be refunded for?\n\n invoice_line_id track_name artist_name purchase_date quantity_purchased price_per_unit\n----------------- -------------------------------- ------------- ------------------- -------------------- ----------------\n 267 How Many More Times Led Zeppelin 2009-08-06 00:00:00 1 0.99\n 268 What Is And What Should Never Be Led Zeppelin 2009-08-06 00:00:00 1 0.99',
+ "trajectory": ["refund_agent", "lookup"],
+ },
+ {
+ "question": "Who recorded Wish You Were Here again? What other albums of there's do you have?",
+ "response": "Wish You Were Here is an album by Pink Floyd",
+ "trajectory": ["question_answering_agent", "lookup_album"],
+ },
+ {
+ "question": "I want a full refund for invoice 237",
+ "response": "You have been refunded $0.99.",
+ "trajectory": ["refund_agent", "refund"],
+ },
]
+#endregion
-dataset_name = "SQL Agent Response"
+dataset_name = "Chinook Customer Service Bot: E2E"
+
+#region
if not client.has_dataset(dataset_name=dataset_name):
dataset = client.create_dataset(dataset_name=dataset_name)
- inputs, outputs = zip(
- *[({"input": text}, {"output": label}) for text, label in examples]
+ client.create_examples(
+ inputs=[{"question": ex["question"]} for ex in examples],
+ outputs=[{"response": ex["response"], "trajectory": ex["trajectory"]} for ex in examples],
+ dataset_id=dataset.id
)
- client.create_examples(inputs=inputs, outputs=outputs, dataset_id=dataset.id)
+#endregion
```
-`Run chain`
+We'll create a custom [LLM-as-judge](../concepts#llm-as-judge) evaluator that uses another model to compare our agent's output on each example to the reference response, and judge if they're equivalent or not:
```python
-def predict_sql_agent_answer(example: dict):
- """Use this for answer evaluation"""
- msg = {"messages": ("user", example["input"])}
- messages = graph.invoke(msg, config)
- return {"response": messages['messages'][-1].content}
+# LLM-as-judge instructions
+#region
+grader_instructions = """You are a teacher grading a quiz.
+
+You will be given a QUESTION, the GROUND TRUTH (correct) RESPONSE, and the STUDENT RESPONSE.
+
+Here is the grade criteria to follow:
+(1) Grade the student responses based ONLY on their factual accuracy relative to the ground truth answer.
+(2) Ensure that the student response does not contain any conflicting statements.
+(3) It is OK if the student response contains more information than the ground truth response, as long as it is factually accurate relative to the ground truth response.
+
+Correctness:
+True means that the student's response meets all of the criteria.
+False means that the student's response does not meet all of the criteria.
+
+Explain your reasoning in a step-by-step manner to ensure your reasoning and conclusion are correct."""
+#endregion
+
+# LLM-as-judge output schema
+#region
+class Grade(TypedDict):
+ """Compare the expected and actual answers and grade the actual answer."""
+ reasoning: Annotated[str, ..., "Explain your reasoning for whether the actual response is correct or not."]
+ is_correct: Annotated[bool, ..., "True if the student response is mostly or exactly correct, otherwise False."]
+#endregion
+
+# Judge LLM
+grader_llm = init_chat_model("gpt-4o-mini", temperature=0).with_structured_output(Grade, method="json_schema", strict=True)
+
+# Evaluator function
+#region
+async def final_answer_correct(inputs: dict, outputs: dict, reference_outputs: dict) -> bool:
+ """Evaluate if the final response is equivalent to reference response."""
+
+ # Note that we assume the outputs has a 'response' dictionary. We'll need to make sure
+ # that the target function we define includes this key.
+ user = f"""QUESTION: {inputs['question']}
+ GROUND TRUTH RESPONSE: {reference_outputs['response']}
+ STUDENT RESPONSE: {outputs['response']}"""
+
+ grade = await grader_llm.ainvoke([{"role": "system", "content": grader_instructions}, {"role": "user", "content": user}])
+ return grade["is_correct"]
+#endregion
```
-`Evaluator`
+Now we can run our evaluation. Our evaluator assumes that our target function returns a 'response' key, so lets define a target function that does so.
-This can [follow what we do for RAG](./rag) where we compare the generated answer with the reference answer.
+Also remember that in our refund graph we made the refund node configurable, so that if we specified `config={"env": "test"}`, we would mock out the refunds without actually updating the DB.
+We'll use this configurable variable in our target `run_graph` method when invoking our graph:
```python
-from langchain import hub
-from langchain_openai import ChatOpenAI
+# Target function
+#region
+async def run_graph(inputs: dict) -> dict:
+ """Run graph and track the trajectory it takes along with the final response."""
+ result = await graph.ainvoke({"messages": [
+ { "role": "user", "content": inputs['question']},
+ ]}, config={"env": "test"})
+ return {"response": result["followup"]}
+#endregion
+
+# Evaluation job and results
+experiment_results = await client.aevaluate(
+ run_graph,
+ data=dataset_name,
+ evaluators=[final_answer_correct],
+ experiment_prefix="sql-agent-gpt4o-e2e",
+ num_repetitions=1,
+ max_concurrency=4,
+)
+experiment_results.to_pandas()
+```
-# Grade prompt
-grade_prompt_answer_accuracy = prompt = hub.pull("langchain-ai/rag-answer-vs-reference")
+You can see what these results look like here: [LangSmith link](https://smith.langchain.com/public/708d08f4-300e-4c75-9677-c6b71b0d28c9/d).
-def answer_evaluator(run, example) -> dict:
- """
- A simple evaluator for RAG answer accuracy
- """
+### Trajectory evaluator
- # Get question, ground truth answer, RAG chain answer
- input_question = example.inputs["input"]
- reference = example.outputs["output"]
- prediction = run.outputs["response"]
+As agents become more complex, they have more potential points of failure. Rather than using simple pass/fail evaluations, it's often better to use evaluations that can give partial credit when an agent takes some correct steps, even if it doesn't reach the right final answer.
- # LLM grader
- llm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
+This is where trajectory evaluations come in. A trajectory evaluation:
+1. Compares the actual sequence of steps the agent took against an expected sequence
+2. Calculates a score based on how many of the expected steps were completed correctly
- # Structured prompt
- answer_grader = grade_prompt_answer_accuracy | llm
+For this example, our end-to-end dataset contains an ordered list of steps that we expect the agent to take. Let's create an evaluator that checks the agent's actual trajectory against these expected steps and calculates what percentage were completed:
- # Run evaluator
- score = answer_grader.invoke({"question": input_question,
- "correct_answer": reference,
- "student_answer": prediction})
- score = score["Score"]
+```python
+#region
+def trajectory_subsequence(outputs: dict, reference_outputs: dict) -> float:
+ """Check how many of the desired steps the agent took."""
+ if len(reference_outputs['trajectory']) > len(outputs['trajectory']):
+ return False
+
+ i = j = 0
+ while i < len(reference_outputs['trajectory']) and j < len(outputs['trajectory']):
+ if reference_outputs['trajectory'][i] == outputs['trajectory'][j]:
+ i += 1
+ j += 1
+
+ return i / len(reference_outputs['trajectory'])
+#endregion
+```
+
+Now we can run our evaluation. Our evaluator assumes that our target function returns a 'trajectory' key, so lets define a target function that does so. We'll need to usage [LangGraph's streaming capabilities](https://langchain-ai.github.io/langgraph/concepts/streaming/) to record the trajectory.
- return {"key": "answer_v_reference_score", "score": score}
+Note that we are reusing the same dataset as for our final response evaluation, so we could have run both evaluators together and defined a target function that returns both "response" and "trajectory".
+In practice it's often useful to have separate datasets for each type of evaluation, which is why we show them separately here:
+
+```python
+#region
+async def run_graph(inputs: dict) -> dict:
+ """Run graph and track the trajectory it takes along with the final response."""
+ trajectory = []
+ # Set subgraph=True to stream events from subgraphs of the main graph: https://langchain-ai.github.io/langgraph/how-tos/streaming-subgraphs/
+ # Set stream_mode="debug" to stream all possible events: https://langchain-ai.github.io/langgraph/concepts/streaming
+ async for namespace, chunk in graph.astream({"messages": [
+ {
+ "role": "user",
+ "content": inputs['question'],
+ }
+ ]}, subgraphs=True, stream_mode="debug"):
+ # Event type for entering a node
+ if chunk['type'] == 'task':
+ # Record the node name
+ trajectory.append(chunk['payload']['name'])
+ # Given how we defined our dataset, we also need to track when specific tools are
+ # called by our question answering ReACT agent. These tool calls can be found
+ # when the ToolsNode (named "tools") is invoked by looking at the AIMessage.tool_calls
+ # of the latest input message.
+ if chunk['payload']['name'] == 'tools' and chunk['type'] == 'task':
+ for tc in chunk['payload']['input']['messages'][-1].tool_calls:
+ trajectory.append(tc['name'])
+
+ return {"trajectory": trajectory}
+#endregion
+
+experiment_results = await client.aevaluate(
+ run_graph,
+ data=dataset_name,
+ evaluators=[trajectory_subsequence],
+ experiment_prefix="sql-agent-gpt4o-trajectory",
+ num_repetitions=1,
+ max_concurrency=4,
+)
+experiment_results.to_pandas()
```
-`Create evaluation`
+You can see what these results look like here: [LangSmith link](https://smith.langchain.com/public/708d08f4-300e-4c75-9677-c6b71b0d28c9/d).
+
+### Single step evaluators
+
+While end-to-end tests give you the most signal about your agents performance, for the sake of debugging and iterating on your agent it can be helpful to pinpoint specific steps that are difficult and evaluate them directly.
+
+In our case, a crucial part of our agent is that it routes the user's intention correctly into either the "refund" path or the "question answering" path. Let's create a dataset and run some evaluations to directly stress test this one component.
```python
-from langsmith import evaluate
+# Create dataset
+#region
+examples = [
+ {"messages": [{"role": "user", "content": "i bought some tracks recently and i dont like them"}], "route": "refund_agent"},
+ {"messages": [{"role": "user", "content": "I was thinking of purchasing some Rolling Stones tunes, any recommendations?"}], "route": "question_answering_agent"},
+ {"messages": [{"role": "user", "content": "i want a refund on purchase 237"}, {"role": "assistant", "content": "I've refunded you a total of $1.98. How else can I help you today?"}, {"role": "user", "content": "did prince release any albums in 2000?"}], "route": "question_answering_agent"},
+ {"messages": [{"role": "user", "content": "i purchased a cover of Yesterday recently but can't remember who it was by, which versions of it do you have?"}], "route": "question_answering_agent"},
+]
+#endregion
-experiment_results = evaluate(
- predict_sql_agent_answer,
+dataset_name = "Chinook Customer Service Bot: Intent Classifier"
+#region
+if not client.has_dataset(dataset_name=dataset_name):
+ dataset = client.create_dataset(dataset_name=dataset_name)
+ client.create_examples(
+ inputs = [{"messages": ex["messages"]} for ex in examples],
+ outputs = [{"route": ex["route"]} for ex in examples],
+ dataset_id=dataset.id
+ )
+#endregion
+
+# Evaluator
+def correct(outputs: dict, reference_outputs: dict) -> bool:
+ """Check if the agent chose the correct route."""
+ return outputs["route"] == reference_outputs["route"]
+
+# Target function for running the relevant step
+async def run_intent_classifier(inputs: dict) -> dict:
+ # Note that we can access and run the intent_classifier node of our graph directly.
+ command = await graph.nodes['intent_classifier'].ainvoke(inputs)
+ return {"route": command.goto}
+
+# Run evaluation
+experiment_results = await client.aevaluate(
+ run_intent_classifier,
data=dataset_name,
- evaluators=[answer_evaluator],
- experiment_prefix=experiment_prefix + "-response-v-reference",
- num_repetitions=3,
- metadata={"version": metadata},
+ evaluators=[correct],
+ experiment_prefix="sql-agent-gpt4o-intent-classifier",
+ max_concurrency=4,
)
```
-### Single step evaluation
+You can see what these results look like here: [LangSmith link](https://smith.langchain.com/public/f133dae2-8a88-43a0-9bfd-ab45bfa3920b/d).
-Agents generally make multiple actions. While it is useful to evaluate them end-to-end, it can also be useful to evaluate the individual actions. This generally involves evaluating a single step of the agent - the LLM call where it decides what to do.
+## Reference code
-:::tip
+Here's a consolidated script with all the above code:
-See the full overview of single step evaluation in our [conceptual guide](../concepts#evaluating-a-single-step-of-an-agent).
+````python
+#region [collapsed]
+import json
+import sqlite3
+from typing import Literal
+
+from langchain.chat_models import init_chat_model
+from langchain.embeddings import init_embeddings
+from langchain_core.runnables import RunnableConfig
+from langchain_core.tools import tool
+from langchain_core.vectorstores import InMemoryVectorStore
+from langgraph.graph import END, StateGraph
+from langgraph.graph.message import AnyMessage, add_messages
+from langgraph.prebuilt import create_react_agent
+from langgraph.types import Command, interrupt
+from langsmith import Client
+import requests
+from tabulate import tabulate
+from typing_extensions import Annotated, TypedDict
-:::
+url = "https://storage.googleapis.com/benchmarks-artifacts/chinook/Chinook.db"
-We can check a specific tool call using [a custom evaluator](../how_to_guides/custom_evaluator):
+response = requests.get(url)
-- Here, we just invoke the assistant, `assistant_runnable`, with a prompt and check if the resulting tool call is as expected.
-- Here, we are using a specialized agent where the tools are hard-coded (rather than passed with the dataset input).
-- We specify the `reference` tool call for the step that we are evaluating, `expected_tool_call`.
+if response.status_code == 200:
+ # Open a local file in binary write mode
+ with open("chinook.db", "wb") as file:
+ # Write the content of the response (the file) to the local file
+ file.write(response.content)
+ print("File downloaded and saved as Chinook.db")
+else:
+ print(f"Failed to download the file. Status code: {response.status_code}")
-```python
-from langsmith.schemas import Example, Run
-def predict_assistant(example: dict):
- """Invoke assistant for single tool call evaluation"""
- msg = [ ("user", example["input"]) ]
- result = assistant_runnable.invoke({"messages":msg})
- return {"response": result}
+def _refund(
+ invoice_id: int | None, invoice_line_ids: list[int] | None, mock: bool = False
+) -> float:
+ """Given an Invoice ID and/or Invoice Line IDs, delete the relevant Invoice/InvoiceLine records in the Chinook DB.
+ Args:
+ invoice_id: The Invoice to delete.
+ invoice_line_ids: The Invoice Lines to delete.
+ mock: If True, do not actually delete the specified Invoice/Invoice Lines. Used for testing purposes.
-def check_specific_tool_call(root_run: Run, example: Example) -> dict:
- """
- Check if the first tool call in the response matches the expected tool call.
+ Returns:
+ float: The total dollar amount that was deleted (or mock deleted).
"""
- # Expected tool call
- expected_tool_call = 'sql_db_list_tables'
- # Run
- response = root_run.outputs["response"]
+ if invoice_id is None and invoice_line_ids is None:
+ return 0.0
+
+ # Connect to the Chinook database
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ total_refund = 0.0
- # Get tool call
try:
- tool_call = getattr(response, 'tool_calls', [])[0]['name']
- except (IndexError, KeyError):
- tool_call = None
+ # If invoice_id is provided, delete entire invoice and its lines
+ if invoice_id is not None:
+ # First get the total amount for the invoice
+ cursor.execute(
+ """
+ SELECT Total
+ FROM Invoice
+ WHERE InvoiceId = ?
+ """,
+ (invoice_id,),
+ )
- score = 1 if tool_call == expected_tool_call else 0
- return {"score": score, "key": "single_tool_call"}
+ result = cursor.fetchone()
+ if result:
+ total_refund += result[0]
+
+ # Delete invoice lines first (due to foreign key constraints)
+ if not mock:
+ cursor.execute(
+ """
+ DELETE FROM InvoiceLine
+ WHERE InvoiceId = ?
+ """,
+ (invoice_id,),
+ )
+
+ # Then delete the invoice
+ cursor.execute(
+ """
+ DELETE FROM Invoice
+ WHERE InvoiceId = ?
+ """,
+ (invoice_id,),
+ )
+
+ # If specific invoice lines are provided
+ if invoice_line_ids is not None:
+ # Get the total amount for the specified invoice lines
+ placeholders = ",".join(["?" for _ in invoice_line_ids])
+ cursor.execute(
+ f"""
+ SELECT SUM(UnitPrice * Quantity)
+ FROM InvoiceLine
+ WHERE InvoiceLineId IN ({placeholders})
+ """,
+ invoice_line_ids,
+ )
-experiment_results = evaluate(
- predict_assistant,
- data=dataset_name,
- evaluators=[check_specific_tool_call],
- experiment_prefix=experiment_prefix + "-single-tool",
- num_repetitions=3,
- metadata={"version": metadata},
+ result = cursor.fetchone()
+ if result and result[0]:
+ total_refund += result[0]
+
+ if not mock:
+ # Delete the specified invoice lines
+ cursor.execute(
+ f"""
+ DELETE FROM InvoiceLine
+ WHERE InvoiceLineId IN ({placeholders})
+ """,
+ invoice_line_ids,
+ )
+
+ # Commit the changes
+ conn.commit()
+
+ except sqlite3.Error as e:
+ # Roll back in case of error
+ conn.rollback()
+ raise e
+
+ finally:
+ # Close the connection
+ conn.close()
+
+ return float(total_refund)
+
+
+def _lookup(
+ customer_first_name: str,
+ customer_last_name: str,
+ customer_phone: str,
+ track_name: str | None,
+ album_title: str | None,
+ artist_name: str | None,
+ purchase_date_iso_8601: str | None,
+) -> list[dict]:
+ """Find all of the Invoice Line IDs in the Chinook DB for the given filters.
+
+ Returns:
+ a list of dictionaries that contain keys: {
+ 'invoice_line_id',
+ 'track_name',
+ 'artist_name',
+ 'purchase_date',
+ 'quantity_purchased',
+ 'price_per_unit'
+ }
+ """
+
+ # Connect to the database
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ # Base query joining all necessary tables
+ query = """
+ SELECT
+ il.InvoiceLineId,
+ t.Name as track_name,
+ art.Name as artist_name,
+ i.InvoiceDate as purchase_date,
+ il.Quantity as quantity_purchased,
+ il.UnitPrice as price_per_unit
+ FROM InvoiceLine il
+ JOIN Invoice i ON il.InvoiceId = i.InvoiceId
+ JOIN Customer c ON i.CustomerId = c.CustomerId
+ JOIN Track t ON il.TrackId = t.TrackId
+ JOIN Album alb ON t.AlbumId = alb.AlbumId
+ JOIN Artist art ON alb.ArtistId = art.ArtistId
+ WHERE c.FirstName = ?
+ AND c.LastName = ?
+ AND c.Phone = ?
+ """
+
+ # Parameters for the query
+ params = [customer_first_name, customer_last_name, customer_phone]
+
+ # Add optional filters
+ if track_name:
+ query += " AND t.Name = ?"
+ params.append(track_name)
+
+ if album_title:
+ query += " AND alb.Title = ?"
+ params.append(album_title)
+
+ if artist_name:
+ query += " AND art.Name = ?"
+ params.append(artist_name)
+
+ if purchase_date_iso_8601:
+ query += " AND date(i.InvoiceDate) = date(?)"
+ params.append(purchase_date_iso_8601)
+
+ # Execute query
+ cursor.execute(query, params)
+
+ # Fetch results
+ results = cursor.fetchall()
+
+ # Convert results to list of dictionaries
+ output = []
+ for row in results:
+ output.append(
+ {
+ "invoice_line_id": row[0],
+ "track_name": row[1],
+ "artist_name": row[2],
+ "purchase_date": row[3],
+ "quantity_purchased": row[4],
+ "price_per_unit": row[5],
+ }
+ )
+
+ # Close connection
+ conn.close()
+
+ return output
+
+
+# Graph state.
+class State(TypedDict):
+ """Agent state."""
+
+ messages: Annotated[list[AnyMessage], add_messages]
+ followup: str | None
+
+ invoice_id: int | None
+ invoice_line_ids: list[int] | None
+ customer_first_name: str | None
+ customer_last_name: str | None
+ customer_phone: str | None
+ track_name: str | None
+ album_title: str | None
+ artist_name: str | None
+ purchase_date_iso_8601: str | None
+
+
+# Instructions for extracting the user/purchase info from the conversation.
+gather_info_instructions = """You are managing an online music store that sells song tracks. \
+Customers can buy multiple tracks at a time and these purchases are recorded in a database as \
+an Invoice per purchase and an associated set of Invoice Lines for each purchased track.
+
+Your task is to help customers who would like a refund for one or more of the tracks they've \
+purchased. In order for you to be able refund them, the customer must specify the Invoice ID \
+to get a refund on all the tracks they bought in a single transaction, or one or more Invoice \
+Line IDs if they would like refunds on individual tracks.
+
+Often a user will not know the specific Invoice ID(s) or Invoice Line ID(s) for which they \
+would like a refund. In this case you can help them look up their invoices by asking them to \
+specify:
+- Required: Their first name, last name, and phone number.
+- Optionally: The track name, artist name, album name, or purchase date.
+
+If the customer has not specified the required information (either Invoice/Invoice Line IDs \
+or first name, last name, phone) then please ask them to specify it."""
+
+
+# Extraction schema, mirrors the graph state.
+class PurchaseInformation(TypedDict):
+ """All of the known information about the invoice / invoice lines the customer would like refunded. Do not make up values, leave fields as null if you don't know their value."""
+
+ invoice_id: int | None
+ invoice_line_ids: list[int] | None
+ customer_first_name: str | None
+ customer_last_name: str | None
+ customer_phone: str | None
+ track_name: str | None
+ album_title: str | None
+ artist_name: str | None
+ purchase_date_iso_8601: str | None
+ followup: Annotated[
+ str | None,
+ ...,
+ "If the user hasn't enough identifying information, please tell them what the required information is and ask them to specify it.",
+ ]
+
+
+# Model for performing extraction.
+info_llm = init_chat_model("gpt-4o-mini").with_structured_output(
+ PurchaseInformation, method="json_schema", include_raw=True
)
-```
-### Trajectory
-We can check a trajectory of tool calls using [custom evaluators](../how_to_guides/custom_evaluator):
+# Graph node for extracting user info and routing to lookup/refund/END.
+async def gather_info(state: State) -> Command[Literal["lookup", "refund", END]]:
+ info = await info_llm.ainvoke(
+ [
+ {"role": "system", "content": gather_info_instructions},
+ *state["messages"],
+ ]
+ )
+ parsed = info["parsed"]
+ if any(parsed[k] for k in ("invoice_id", "invoice_line_ids")):
+ goto = "refund"
+ elif all(
+ parsed[k]
+ for k in ("customer_first_name", "customer_last_name", "customer_phone")
+ ):
+ goto = "lookup"
+ else:
+ goto = END
+ update = {"messages": [info["raw"]], **parsed}
+ return Command(update=update, goto=goto)
+
+
+# Graph node for executing the refund.
+# Note that here we inspect the runtime config for an "env" variable.
+# If "env" is set to "test", then we don't actually delete any rows from our database.
+# This will become important when we're running our evaluations.
+def refund(state: State, config: RunnableConfig) -> dict:
+ # Whether to mock the deletion. True if the configurable var 'env' is set to 'test'.
+ mock = config.get("configurable", {}).get("env", "prod") == "test"
+ refunded = _refund(
+ invoice_id=state["invoice_id"],
+ invoice_line_ids=state["invoice_line_ids"],
+ mock=mock,
+ )
+ response = f"You have been refunded a total of: ${refunded:.2f}. Is there anything else I can help with?"
+ return {
+ "messages": [{"role": "assistant", "content": response}],
+ "followup": response,
+ }
-- Here, we just invoke the agent, `graph.invoke`, with a prompt.
-- Here, we are using a specialized agent where the tools are hard-coded (rather than passed with the dataset input).
-- We extract the list of tools called, using `find_tool_calls`.
-- Custom functions can process these tool calls in various user-defined ways.
-- We can check if all expected tools are called in any order: `contains_all_tool_calls_any_order`
-- We can check if all expected tools are called in order, allowing for insertion of tool calls: `contains_all_tool_calls_in_order`
-- We can check if all expected tools are called in the exact order: `contains_all_tool_calls_in_order_exact_match`
-:::tip
+# Graph node for looking up the users purchases
+def lookup(state: State) -> dict:
+ args = (
+ state[k]
+ for k in (
+ "customer_first_name",
+ "customer_last_name",
+ "customer_phone",
+ "track_name",
+ "album_title",
+ "artist_name",
+ "purchase_date_iso_8601",
+ )
+ )
+ results = _lookup(*args)
+ if not results:
+ response = "We did not find any purchases associated with the information you've provided. Are you sure you've entered all of your information correctly?"
+ followup = response
+ else:
+ response = f"Which of the following purchases would you like to be refunded for?\n\n```json{json.dumps(results, indent=2)}\n```"
+ followup = f"Which of the following purchases would you like to be refunded for?\n\n{tabulate(results, headers='keys')}"
+ return {
+ "messages": [{"role": "assistant", "content": response}],
+ "followup": followup,
+ "invoice_line_ids": [res["invoice_line_id"] for res in results],
+ }
-See the full overview of single step evaluation in our [conceptual guide](../concepts#evaluating-an-agents-trajectory).
-:::
+# Building our graph
+graph_builder = StateGraph(State)
-```python
-def predict_sql_agent_messages(example: dict):
- """Use this for answer evaluation"""
- msg = {"messages": ("user", example["input"])}
- messages = graph.invoke(msg, config)
- return {"response": messages}
+graph_builder.add_node(gather_info)
+graph_builder.add_node(refund)
+graph_builder.add_node(lookup)
-def find_tool_calls(messages):
- """
- Find all tool calls in the messages returned
- """
- tool_calls = [tc['name'] for m in messages['messages'] for tc in getattr(m, 'tool_calls', [])]
- return tool_calls
+graph_builder.set_entry_point("gather_info")
+graph_builder.add_edge("lookup", END)
+graph_builder.add_edge("refund", END)
+
+refund_graph = graph_builder.compile()
+
+
+# Our SQL queries will only work if we filter on the exact string values that are in the DB.
+# To ensure this, we'll create vectorstore indexes for all of the artists, tracks and albums
+# ahead of time and use those to disambiguate the user input. E.g. if a user searches for
+# songs by "prince" and our DB records the artist as "Prince", ideally when we query our
+# artist vectorstore for "prince" we'll get back the value "Prince", which we can then
+# use in our SQL queries.
+def index_fields() -> (
+ tuple[InMemoryVectorStore, InMemoryVectorStore, InMemoryVectorStore]
+):
+ """Create an index for all artists, an index for all albums, and an index for all songs."""
+ try:
+ # Connect to the chinook database
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ # Fetch all results
+ tracks = cursor.execute("SELECT Name FROM Track").fetchall()
+ artists = cursor.execute("SELECT Name FROM Artist").fetchall()
+ albums = cursor.execute("SELECT Title FROM Album").fetchall()
+ finally:
+ # Close the connection
+ if conn:
+ conn.close()
+
+ embeddings = init_embeddings("openai:text-embedding-3-small")
+
+ track_store = InMemoryVectorStore(embeddings)
+ artist_store = InMemoryVectorStore(embeddings)
+ album_store = InMemoryVectorStore(embeddings)
+
+ track_store.add_texts([t[0] for t in tracks])
+ artist_store.add_texts([a[0] for a in artists])
+ album_store.add_texts([a[0] for a in albums])
+ return track_store, artist_store, album_store
-def contains_all_tool_calls_any_order(root_run: Run, example: Example) -> dict:
+
+track_store, artist_store, album_store = index_fields()
+
+
+# Agent tools
+@tool
+def lookup_track(
+ track_name: str | None = None,
+ album_title: str | None = None,
+ artist_name: str | None = None,
+) -> list[dict]:
+ """Lookup a track in Chinook DB based on identifying information about.
+
+ Returns:
+ a list of dictionaries per matching track that contain keys {'track_name', 'artist_name', 'album_name'}
"""
- Check if all expected tools are called in any order.
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ query = """
+ SELECT DISTINCT t.Name as track_name, ar.Name as artist_name, al.Title as album_name
+ FROM Track t
+ JOIN Album al ON t.AlbumId = al.AlbumId
+ JOIN Artist ar ON al.ArtistId = ar.ArtistId
+ WHERE 1=1
"""
- expected = ['sql_db_list_tables', 'sql_db_schema', 'sql_db_query_checker', 'sql_db_query', 'check_result']
- messages = root_run.outputs["response"]
- tool_calls = find_tool_calls(messages)
- # Optionally, log the tool calls -
- #print("Here are my tool calls:")
- #print(tool_calls)
- if set(expected) <= set(tool_calls):
- score = 1
- else:
- score = 0
- return {"score": int(score), "key": "multi_tool_call_any_order"}
+ params = []
+
+ if track_name:
+ track_name = track_store.similarity_search(track_name, k=1)[0].page_content
+ query += " AND t.Name LIKE ?"
+ params.append(f"%{track_name}%")
+ if album_title:
+ album_title = album_store.similarity_search(album_title, k=1)[0].page_content
+ query += " AND al.Title LIKE ?"
+ params.append(f"%{album_title}%")
+ if artist_name:
+ artist_name = artist_store.similarity_search(artist_name, k=1)[0].page_content
+ query += " AND ar.Name LIKE ?"
+ params.append(f"%{artist_name}%")
+
+ cursor.execute(query, params)
+ results = cursor.fetchall()
+
+ tracks = [
+ {"track_name": row[0], "artist_name": row[1], "album_name": row[2]}
+ for row in results
+ ]
-def contains_all_tool_calls_in_order(root_run: Run, example: Example) -> dict:
+ conn.close()
+ return tracks
+
+
+@tool
+def lookup_album(
+ track_name: str | None = None,
+ album_title: str | None = None,
+ artist_name: str | None = None,
+) -> list[dict]:
+ """Lookup an album in Chinook DB based on identifying information about.
+
+ Returns:
+ a list of dictionaries per matching album that contain keys {'album_name', 'artist_name'}
"""
- Check if all expected tools are called in exact order.
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ query = """
+ SELECT DISTINCT al.Title as album_name, ar.Name as artist_name
+ FROM Album al
+ JOIN Artist ar ON al.ArtistId = ar.ArtistId
+ LEFT JOIN Track t ON t.AlbumId = al.AlbumId
+ WHERE 1=1
"""
- messages = root_run.outputs["response"]
- tool_calls = find_tool_calls(messages)
- # Optionally, log the tool calls -
- #print("Here are my tool calls:")
- #print(tool_calls)
- it = iter(tool_calls)
- expected = ['sql_db_list_tables', 'sql_db_schema', 'sql_db_query_checker', 'sql_db_query', 'check_result']
- if all(elem in it for elem in expected):
- score = 1
- else:
- score = 0
- return {"score": int(score), "key": "multi_tool_call_in_order"}
+ params = []
+
+ if track_name:
+ query += " AND t.Name LIKE ?"
+ params.append(f"%{track_name}%")
+ if album_title:
+ query += " AND al.Title LIKE ?"
+ params.append(f"%{album_title}%")
+ if artist_name:
+ query += " AND ar.Name LIKE ?"
+ params.append(f"%{artist_name}%")
+
+ cursor.execute(query, params)
+ results = cursor.fetchall()
+
+ albums = [{"album_name": row[0], "artist_name": row[1]} for row in results]
+
+ conn.close()
+ return albums
+
-def contains_all_tool_calls_in_order_exact_match(root_run: Run, example: Example) -> dict:
+@tool
+def lookup_artist(
+ track_name: str | None = None,
+ album_title: str | None = None,
+ artist_name: str | None = None,
+) -> list[str]:
+ """Lookup an album in Chinook DB based on identifying information about.
+
+ Returns:
+ a list of matching artist names
"""
- Check if all expected tools are called in exact order and without any additional tool calls.
+ conn = sqlite3.connect("chinook.db")
+ cursor = conn.cursor()
+
+ query = """
+ SELECT DISTINCT ar.Name as artist_name
+ FROM Artist ar
+ LEFT JOIN Album al ON al.ArtistId = ar.ArtistId
+ LEFT JOIN Track t ON t.AlbumId = al.AlbumId
+ WHERE 1=1
"""
- expected = ['sql_db_list_tables', 'sql_db_schema', 'sql_db_query_checker', 'sql_db_query', 'check_result']
- messages = root_run.outputs["response"]
- tool_calls = find_tool_calls(messages)
- # Optionally, log the tool calls -
- #print("Here are my tool calls:")
- #print(tool_calls)
- if tool_calls == expected:
- score = 1
- else:
- score = 0
+ params = []
+
+ if track_name:
+ query += " AND t.Name LIKE ?"
+ params.append(f"%{track_name}%")
+ if album_title:
+ query += " AND al.Title LIKE ?"
+ params.append(f"%{album_title}%")
+ if artist_name:
+ query += " AND ar.Name LIKE ?"
+ params.append(f"%{artist_name}%")
+
+ cursor.execute(query, params)
+ results = cursor.fetchall()
+
+ artists = [row[0] for row in results]
+
+ conn.close()
+ return artists
+
+
+# Agent model
+qa_llm = init_chat_model("claude-3-5-sonnet-latest")
+# The prebuilt ReACT agent only expects State to have a 'messages' key, so the
+# state we defined for the refund agent can also be passed to our lookup agent.
+qa_graph = create_react_agent(qa_llm, [lookup_track, lookup_artist, lookup_album])
- return {"score": int(score), "key": "multi_tool_call_in_exact_order"}
-experiment_results = evaluate(
- predict_sql_agent_messages,
+# Schema for routing user intent.
+# We'll use structured outputs to enforce that the model returns only
+# the desired output.
+class UserIntent(TypedDict):
+ """The user's current intent in the conversation"""
+
+ intent: Literal["refund", "question_answering"]
+
+
+# Routing model with structured output
+router_llm = init_chat_model("gpt-4o-mini").with_structured_output(
+ UserIntent, method="json_schema", strict=True
+)
+
+# Instructions for routing.
+route_instructions = """You are managing an online music store that sells song tracks. \
+You can help customers in two types of ways: (1) answering general questions about \
+tracks sold at your store, (2) helping them get a refund on a purhcase they made at your store.
+
+Based on the following conversation, determine if the user is currently seeking general \
+information about song tracks or if they are trying to refund a specific purchase.
+
+Return 'refund' if they are trying to get a refund and 'question_answering' if they are \
+asking a general music question. Do NOT return anything else. Do NOT try to respond to \
+the user.
+"""
+
+
+# Node for routing.
+async def intent_classifier(
+ state: State,
+) -> Command[Literal["refund_agent", "question_answering_agent"]]:
+ response = router_llm.invoke(
+ [{"role": "system", "content": route_instructions}, *state["messages"]]
+ )
+ return Command(goto=response["intent"] + "_agent")
+
+
+# Node for making sure the 'followup' key is set before our agent run completes.
+def compile_followup(state: State) -> dict:
+ """Set the followup to be the last message if it hasn't explicitly been set."""
+ if not state.get("followup"):
+ return {"followup": state["messages"][-1].content}
+ return {}
+
+
+# Agent definition
+graph_builder = StateGraph(State)
+graph_builder.add_node(intent_classifier)
+# Since all of our subagents have compatible state,
+# we can add them as nodes directly.
+graph_builder.add_node("refund_agent", refund_graph)
+graph_builder.add_node("question_answering_agent", qa_graph)
+graph_builder.add_node(compile_followup)
+
+graph_builder.set_entry_point("intent_classifier")
+graph_builder.add_edge("refund_agent", "compile_followup")
+graph_builder.add_edge("question_answering_agent", "compile_followup")
+graph_builder.add_edge("compile_followup", END)
+
+graph = graph_builder.compile()
+
+
+client = Client()
+
+# Create a dataset
+examples = [
+ {
+ "question": "How many songs do you have by James Brown",
+ "response": "We have 20 songs by James Brown",
+ "trajectory": ["question_answering_agent", "lookup_tracks"],
+ },
+ {
+ "question": "My name is Aaron Mitchell and I'd like a refund.",
+ "response": "I need some more information to help you with the refund. Please specify your phone number, the invoice ID, or the line item IDs for the purchase you'd like refunded.",
+ "trajectory": ["refund_agent"],
+ },
+ {
+ "question": "My name is Aaron Mitchell and I'd like a refund on my Led Zeppelin purchases. My number is +1 (204) 452-6452",
+ "response": "Which of the following purchases would you like to be refunded for?\n\n invoice_line_id track_name artist_name purchase_date quantity_purchased price_per_unit\n----------------- -------------------------------- ------------- ------------------- -------------------- ----------------\n 267 How Many More Times Led Zeppelin 2009-08-06 00:00:00 1 0.99\n 268 What Is And What Should Never Be Led Zeppelin 2009-08-06 00:00:00 1 0.99",
+ "trajectory": ["refund_agent", "lookup"],
+ },
+ {
+ "question": "Who recorded Wish You Were Here again? What other albums of there's do you have?",
+ "response": "Wish You Were Here is an album by Pink Floyd",
+ "trajectory": ["question_answering_agent", "lookup_album"],
+ },
+ {
+ "question": "I want a full refund for invoice 237",
+ "response": "You have been refunded $2.97.",
+ "trajectory": ["refund_agent", "refund"],
+ },
+]
+
+dataset_name = "Chinook Customer Service Bot: E2E"
+
+if not client.has_dataset(dataset_name=dataset_name):
+ dataset = client.create_dataset(dataset_name=dataset_name)
+ client.create_examples(
+ inputs=[{"question": ex["question"]} for ex in examples],
+ outputs=[
+ {"response": ex["response"], "trajectory": ex["trajectory"]}
+ for ex in examples
+ ],
+ dataset_id=dataset.id,
+ )
+
+# LLM-as-judge instructions
+grader_instructions = """You are a teacher grading a quiz.
+
+You will be given a QUESTION, the GROUND TRUTH (correct) RESPONSE, and the STUDENT RESPONSE.
+
+Here is the grade criteria to follow:
+(1) Grade the student responses based ONLY on their factual accuracy relative to the ground truth answer.
+(2) Ensure that the student response does not contain any conflicting statements.
+(3) It is OK if the student response contains more information than the ground truth response, as long as it is factually accurate relative to the ground truth response.
+
+Correctness:
+True means that the student's response meets all of the criteria.
+False means that the student's response does not meet all of the criteria.
+
+Explain your reasoning in a step-by-step manner to ensure your reasoning and conclusion are correct."""
+
+
+# LLM-as-judge output schema
+class Grade(TypedDict):
+ """Compare the expected and actual answers and grade the actual answer."""
+
+ reasoning: Annotated[
+ str,
+ ...,
+ "Explain your reasoning for whether the actual response is correct or not.",
+ ]
+ is_correct: Annotated[
+ bool,
+ ...,
+ "True if the student response is mostly or exactly correct, otherwise False.",
+ ]
+
+
+# Judge LLM
+grader_llm = init_chat_model("gpt-4o-mini", temperature=0).with_structured_output(
+ Grade, method="json_schema", strict=True
+)
+
+
+# Evaluator function
+async def final_answer_correct(
+ inputs: dict, outputs: dict, reference_outputs: dict
+) -> bool:
+ """Evaluate if the final response is equivalent to reference response."""
+
+ # Note that we assume the outputs has a 'response' dictionary. We'll need to make sure
+ # that the target function we define includes this key.
+ user = f"""QUESTION: {inputs['question']}
+ GROUND TRUTH RESPONSE: {reference_outputs['response']}
+ STUDENT RESPONSE: {outputs['response']}"""
+
+ grade = await grader_llm.ainvoke(
+ [
+ {"role": "system", "content": grader_instructions},
+ {"role": "user", "content": user},
+ ]
+ )
+ return grade["is_correct"]
+
+
+# Target function
+async def run_graph(inputs: dict) -> dict:
+ """Run graph and track the trajectory it takes along with the final response."""
+ result = await graph.ainvoke(
+ {
+ "messages": [
+ {"role": "user", "content": inputs["question"]},
+ ]
+ },
+ config={"env": "test"},
+ )
+ return {"response": result["followup"]}
+
+
+# Evaluation job and results
+experiment_results = await client.aevaluate(
+ run_graph,
data=dataset_name,
- evaluators=[contains_all_tool_calls_any_order,contains_all_tool_calls_in_order,contains_all_tool_calls_in_order_exact_match],
- experiment_prefix=experiment_prefix + "-trajectory",
- num_repetitions=3,
- metadata={"version": metadata},
+ evaluators=[final_answer_correct],
+ experiment_prefix="sql-agent-gpt4o-e2e",
+ num_repetitions=1,
+ max_concurrency=4,
)
-```
+experiment_results.to_pandas()
+
+
+def trajectory_subsequence(outputs: dict, reference_outputs: dict) -> float:
+ """Check how many of the desired steps the agent took."""
+ if len(reference_outputs["trajectory"]) > len(outputs["trajectory"]):
+ return False
+
+ i = j = 0
+ while i < len(reference_outputs["trajectory"]) and j < len(outputs["trajectory"]):
+ if reference_outputs["trajectory"][i] == outputs["trajectory"][j]:
+ i += 1
+ j += 1
+
+ return i / len(reference_outputs["trajectory"])
+
+
+async def run_graph(inputs: dict) -> dict:
+ """Run graph and track the trajectory it takes along with the final response."""
+ trajectory = []
+ # Set subgraph=True to stream events from subgraphs of the main graph: https://langchain-ai.github.io/langgraph/how-tos/streaming-subgraphs/
+ # Set stream_mode="debug" to stream all possible events: https://langchain-ai.github.io/langgraph/concepts/streaming
+ async for namespace, chunk in graph.astream(
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": inputs["question"],
+ }
+ ]
+ },
+ subgraphs=True,
+ stream_mode="debug",
+ ):
+ # Event type for entering a node
+ if chunk["type"] == "task":
+ # Record the node name
+ trajectory.append(chunk["payload"]["name"])
+ # Given how we defined our dataset, we also need to track when specific tools are
+ # called by our question answering ReACT agent. These tool calls can be found
+ # when the ToolsNode (named "tools") is invoked by looking at the AIMessage.tool_calls
+ # of the latest input message.
+ if chunk["payload"]["name"] == "tools" and chunk["type"] == "task":
+ for tc in chunk["payload"]["input"]["messages"][-1].tool_calls:
+ trajectory.append(tc["name"])
+
+ return {"trajectory": trajectory}
+
+
+experiment_results = await client.aevaluate(
+ run_graph,
+ data=dataset_name,
+ evaluators=[trajectory_subsequence],
+ experiment_prefix="sql-agent-gpt4o-trajectory",
+ num_repetitions=1,
+ max_concurrency=4,
+)
+experiment_results.to_pandas()
-You can see the results from the evaluations logged to the dataset!
+# Create dataset
+examples = [
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": "i bought some tracks recently and i dont like them",
+ }
+ ],
+ "route": "refund_agent",
+ },
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": "I was thinking of purchasing some Rolling Stones tunes, any recommendations?",
+ }
+ ],
+ "route": "question_answering_agent",
+ },
+ {
+ "messages": [
+ {"role": "user", "content": "i want a refund on purchase 237"},
+ {
+ "role": "assistant",
+ "content": "I've refunded you a total of $1.98. How else can I help you today?",
+ },
+ {"role": "user", "content": "did prince release any albums in 2000?"},
+ ],
+ "route": "question_answering_agent",
+ },
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": "i purchased a cover of Yesterday recently but can't remember who it was by, which versions of it do you have?",
+ }
+ ],
+ "route": "question_answering_agent",
+ },
+]
-[https://smith.langchain.com/public/20808486-67c3-4e30-920b-6d49d6f2b6b8/d](https://smith.langchain.com/public/20808486-67c3-4e30-920b-6d49d6f2b6b8/d)
+dataset_name = "Chinook Customer Service Bot: Intent Classifier"
+if not client.has_dataset(dataset_name=dataset_name):
+ dataset = client.create_dataset(dataset_name=dataset_name)
+ client.create_examples(
+ inputs=[{"messages": ex["messages"]} for ex in examples],
+ outputs=[{"route": ex["route"]} for ex in examples],
+ dataset_id=dataset.id,
+ )
+
+
+# Evaluator
+def correct(outputs: dict, reference_outputs: dict) -> bool:
+ """Check if the agent chose the correct route."""
+ return outputs["route"] == reference_outputs["route"]
+
+
+# Target function for running the relevant step
+async def run_intent_classifier(inputs: dict) -> dict:
+ # Note that we can access and run the intent_classifier node of our graph directly.
+ command = await graph.nodes["intent_classifier"].ainvoke(inputs)
+ return {"route": command.goto}
+
+
+# Run evaluation
+experiment_results = await client.aevaluate(
+ run_intent_classifier,
+ data=dataset_name,
+ evaluators=[correct],
+ experiment_prefix="sql-agent-gpt4o-intent-classifier",
+ max_concurrency=4,
+)
+experiment_results.to_pandas()
+#endregion
+````
diff --git a/docs/evaluation/tutorials/index.mdx b/docs/evaluation/tutorials/index.mdx
index 18da8297..9e313600 100644
--- a/docs/evaluation/tutorials/index.mdx
+++ b/docs/evaluation/tutorials/index.mdx
@@ -5,4 +5,4 @@ New to LangSmith or to LLM app development in general? Read this material to qui
- [Evaluate your LLM application](./tutorials/evaluation)
- [RAG Evaluations](./tutorials/rag)
- [Backtesting](./tutorials/backtesting)
-- [Agent Evaluations](./tutorials/agents)
+- [Evaluate an agent](./tutorials/agents)
diff --git a/docs/evaluation/tutorials/static/agent_tutorial_graph.png b/docs/evaluation/tutorials/static/agent_tutorial_graph.png
new file mode 100644
index 0000000000000000000000000000000000000000..181a69aba5ed9dc1f7489c3f03744d5bcf46af72
GIT binary patch
literal 37935
zcmbSzby$>N*X}5abP7ldN=Qm~BPiV+f;7_IC?F!;l0$b$cPJs<4Fe)Qz|dW1^Lx+x
zz2`gE_5E><*LXS4Jhk`Qd*5rVd)?a*B?W0rG!irj1cLeQt)vPB^6(4zTY&Nq{9>w5
zXaW9t;G`lg4k;TV+k!xzL*7Y>skx`_z|2LTYR)77slrTH^wLz@!F)=<_~udPm$y&D
zgh;|;zLLk!7|VV}w%%!B)Y!o!sGyzcDmylCn?1ex?p<7-^^irNs_EX%cMaJ3bmicnXVX*J6QcYGXpc8^|s?;Zo;%>mS%oos{|5xqtk=e4`tZ>`|7eVijilsfa>peva$8>Kp-dNSad2gYPUf75lSoG`_e~jmTt;RTEj&=4-*z
zIX*nIqdlq8ZiEoejF(8S=T=g>h*=+}Nic0~KI?tbzHxuSbj^o`
zwXPUjqhuF8xB1VSw@bPy^Oob}k3?tp_1Ga2LM{3#ag1Zto^mLtP(=0i8IJ{%KeG5c
z7O-
z&=bh8Lspu7bXa0>gOMAZpJb|DrLSv7u8yQV6fFeZdbVNYs2u#B24<|UqH?%7`l6Bh
z?qMrMQ+(Z43u91Cg%iZ={@n><;=S{9ri7<5BzJsFOiaSU>1Bg$1KmM)n`REoyJY^K
z?oG6I(SnzlcPZo5^zGZXWSh4_BHAm1#lO0P45QSk=+(oCxgf3Npg~n8txqScl^JO~
z#`EMKJa}MWV9*hat*D?t8~Et){DKYj6x
zG%%-
z=jTOyuiZPoV1D`XWqEmdLP17GMx#uR&jmhlc<6|Y9SPpY+sY!qs`~cpAqy#F1cip!
zUt0!^##8DWr2;!Mb2ttj3CUcEc9p+B#L6gJD!g8=tf9dpl#scAh78O6XnnBc4r&75
zov+&%N{dZg1vcB<+@zIH_O^8j8s}JD&(Dg-NPY;ODi&y}l1>=fp?30b35>KOIpcQlee^O>yRIXQS)
z*_w}G!a?CpKU&hB`=zpSmT9bcA^k|JO+k^v0G_M!XwWZDpulsoQ>24CFS
zCH|&>GqHcY0A>;gfd<52v_I!Z-ua~$UE*rX=IZKd(CU3rq+YC|s`|m$c(q?7)ywSe
z_Qu1*gHf|=XR*Z#3{Gijn{bwSse-<$-t)1spGIh$k8h_Re19_?rK|I@EL=zxo5{o7
z-Fat5S-3SgB0@o7=t`^-4Gm3BPHr{6J|p8#Kfzh{AVFEl+br|1AtYj|8K6JfWbmyx
zf_Eq31kLkTXM2iC>@=L5YMPp7@X6wE60WR-x{tH$R8&S_mI0d$CUZHtxh++h4}-QQ
z^VsbwhD+vGE7lPIi1*n(*X@q5?y~)8!C<*NfS#ojUO#
zYX2le&zo|0Z2pAOUt4OBbql5ipw
zYg~PTWHcUfy!p+OgPFW8>4tN;mZzDz5W{Bw?FCE>68DIx?7{c0&Tr<%x$1Fr+?+Q%
zzd88%`PtYG#@947G?J2%pe#J>>_sin1#5Wt_}SUnBKj<`y(a9$Df;hZ&w8R?Pzsxz
zL<0=7MI;MI&{#aLz6XQe-M>U@<#=~Gc51#>KT=np7wXTk))aa#Pzd`
zrd%LcYYc$Z>^)fZbqsTg6#s?Fl1ys|H`Bs6Mh0fGViX)8Ctq4kAW}`9FExk1&o^h|
z!o=cM_miuEK}*PLwy++GBHeed)JL)Edf=PvlgceDmF-8lW6Vy5+?IBMo){$Pr6v#p
z`#me=v4hl8vGs64%&nm;cf9DGzUb6zIpt|3SqTn|1V}PEa2Z@EGBf}}&027|?Ro4i
zv+^nQOD%`{#?#SlQ~Ru!#!Ect-b8L=W>|qHtGzRd;9ZmX8S0h$5zE1Oe8sZj7^4B1
zRx*xL&XEO-=vQ>w!>;*J1k0uvd(+R}^(s
zw6MrsR4F?dQbN0H&Ilp_(^<{UdeqVP?&DvlF)%PjWRR8EfzL0HbZ}tP>4nIEhC||i
zR9APBPyg?Op1!_c|2}}UqtN$4lu*gZ$+<~?KY`@_lE#8CC>t2C#20l3MnbHRXow*U
znfdv_!5NS^jQay+9i4CgK7jb+0=s75<>lq+u_Pj%LXscfUlN^|NJkjDrSKp_8=Qtf
zo;|q#)yT*wGdK4Q@REa{?;F)KG*nhl0KBBX*}npp`&gwQFg~7uMGuLd_@DlBaNvW*
z_0Nm{{Vqa6LP3hi|GXsb08%p^^D<>Xs;G~QWydQ
z(*D%f*LQAiE-)%9i`vG<25wCe>Gr4nL1XrV0+H0mNbO$T-QB}bk^EK`X6D>h6m;}#
zuBZNa?LpQMcUDf$I4k0Q@OaE;ul-vkAtaAL(-+ZEQ7dX{ZtUM(o$iv8k!{L8$`Aw1
z`{U<5qf}c}MapS`0}Fw0J_a_tj=a6OnxCInHFVn?5xeDrOmG7;pKDoI6eJ`hq@}^y
zKOt=mJ5Ol|$P?jbkW!os75?!pEd=ie6^Ox*oFVdhr3
zUELEJ?H}D)h(#X^A$nwP@@MAWBo3O6raczUD<1B3S1-#QwB50s=^2j#u_QD|gT0+6
zy0<4&eNQj>SuN^)Am<$-C&jWi94GTG_Ozg{UCDw5$vgIi^%$I!$Ms`@tQ$^2Oz0Im
zoO{xGrrRI!91`EYs}JEiyL3pZWg$Az3UWrNZS*TuqRka}B5DfyZpCs{swDfQ4xbPvLQsE(
zAGAw?wE%&H`GeW{uZ8jde~_Uz49!PAgD1PM{nDHs=y|I+{Pyb9&3k`8U5UQrU_B`4
zv3?6|1$*!6^kv)%-CwV>liq=Ws69VK^OjIRQy?ZrUw=PAXxDPURf!IHaX}rD5*28j
zOxK#@$q8mX86r{BEAiCzz5D*kZ|v5CT|5N}VgcK)qDr^nSIWlwR#BT+uG)4^M|YD@-kZ7P*e8Vx;9u{
zb)!+z?NEjAN{ol0+Qk!ewZpY!QJ@^jj$<
z-#lk9m4EIv_FNp0o;<#CJ;R_}Fz=so!3$ZSFuZ()9QYK`3NxEG5NH_43s#hncSrZXb|;!Pxfu^T$@a2V65@C(1;apR=$F
z-!)KCskFWM23iZoGtWFnxx8%Yhf3>wJwe;q)&}WJ)%YeG?#2)_E5WedJr5Yc_-_!D
z7UOIe(cq+6Q_KrN%6y4Luh>sWku7z5yc`pZy|+7ClAd%F98gDFh_8MnVog`Rnyqrvm6DqK
zd{ME|Eu~-YZZ>~<5*Sr$!ykBxO&560E#m62(+Cc|sZu1VlZ0P54lmk}3%I6`aTPn!
z?eW+dI`F1OYT(OuMpD4`2*}vigWpO>WM80XQ3^LIqRdt}t@Or9OI%`J8%>;NWr>rM
zr&U%~e&LsRA^vu$)re8^Mq5kG!o)(YEuGXQzR7Z&aKcKru9Sm~
z`^Kl2p~(2+@=+i*!sqz>C$w7%SA1ueJYO-T-?)OLk*hV%9nn2#o$dGiy|J;CwY4IO
z<;;(e=;(vQD*a~X(;-+&Y?c}F@#aa;%r}Klok1kIZQq$l{hNQ$KHFnb>IDdE4eius+S!HSl
zvmDBfB(gxSc-Dr7cBKafnF|;f*Vat0uy^90QY*V^tosR!uJLSut
zuU;uL5n^LtC$L7ped+0Wd1$9wW2Fl7dHDL?`>O8O3yOc0ymUBYrePJS({GB%mbxBx
z*~8u*559`_HKQ|(f+kwqnjLJ#!y^}3oj&wUKy_;Ep@NpR>1B&0-`}Yu#o>(A%3m!J
zGU>L^x}Nv^X!l1l2%%J1D3xd`+sNl^VV_Ju6#jY@DReLPq}b
z`0<5?2KT)om(WCAmQPtAv}052szeoQvN|o`|m*d@Py#p8TMWExA
zxzZjU7qd=uYqFijY*c~$Yq;XyLPHCJg5EqXbJN%Xl2%^
zhYEVEo`<+gAZ52))e#X9wRj)M(Z-VV)^GPshPDbnXlR(_C(z#!l<;|;kD?Zm4H?a*=#d=eik&QmJ!=Pq
z1Cgb+uhtISWeH1B&~}^<3%w$@diGZej`qViEJT>7O1exzVK|R6l2~>Ir#MC(j|x_i
zE#H@@rYw0Nf1hiBs~PEo82W;pI7v#G?O>FPx5F`V`)YKH+6A+4iEJRP&zkYLcAWw0?81!HwtSHT1SvFVMLd(s~_4M>?a@)II0gbx7+5;2^
zfC9BOc2s}k%$EU;py>?`D^e-kHs{{0b#-n^N=j;Kaav64jt$PzNO}07T^AP@AY+M+
zjt&Y6s;H;{If`@eLrjo}!83AlRxeMs2UB>!A4+zUuMG_s%^pYR-omg3(V!bZXGKIr
zfD{GfZ@0I%5??Sf1#k;QK$p}XetQ1uRiQ1c5eo|o{9aLAoyztBd5$x~es8g*$!SAD
zNofR_@@#jWh>UC@PoA_<;*%$cyw#}*L(2_X`j+kaoi~4&^u~mRg}qYE)6&ryOGD@U
z48YLm@_2i?TsoX2J~j3FZ&zr0yQni?z(degw0hqgPr!45M~C3k0dkPbq$g^9FlC$^
zjZ+6uRbMbkKYsj3O-&sW6C)=t51yjvMz7>S-`lPP##~lj9`KoGpi$Fh`tx;mhLn_)
zjS>&ks9(LZoUJrn@;TGDx3{;lDk?5!goTKzgWxTo+56&f$?pym3(FUzTVcc;p2wTG
zgoILV^!^N)=_!{tXG{2acvuvI1;xevW-C-K1r%{68QMlh%V`2`jm}#tYHFO$n@Wa;
zOQYG+{U+>$=;M6;G9bJkpPXECY!DU}rlO((v|KhYD@JPP10lp>i)?toWWFLal!z5i
z{P^VLY;Q@#^VmWh1{u`1mqTceO*dOkpLe*AX;F#807wI?-Uj5YHJgE
zP00D3?cLn&w#(aI;-Ln2WC{?8?#A?*fYl}|D@)2{r3NTeOUuIid^%XBe~3~{=2VCz
zHAN&51%-iCs%ecblI5O-LXhj&T
zT0j1bWJ&-Il#P?q`{uGHJNpH!0rE_Y{wavGfKze2I9zMHy)15O5`cx=^95o*BPfI#
z!AkQuT5o73mLKey=R&Xk=%+lsGXl
zH>abeJ>43oqoFYc?CJfq2tdZ%#Cn`z`{?0=;0|1rM<2Swi1{2%0A0ML@Td=@jOsCI
zX*O0?GAO8khCxAjCgLl2pM(F(mX5StY8A5o)4sZDA|W9G7c*DVr0?CQF|PRDzb5tO
z4Va93o1@wH1PaLS8MwGy07wCfT0B%cJoBZ*+AWx1lf{~M`yJR|jrxJ10Yo@>DdeSy
z$Wmvg$y~MN34D^2-MJJi#J+MnR$A
zMyLddGm+O+QNbf1ScAh!2nc9!Q0`G81d`ziKwuJpKj3hAL6k5*KTiI{3QHo$q5xzL
zpz(>|ZE04jG2T>lAun&c%No%!;wAS7fAQA}hdJ5ZWv
z0%e3s)Ba7Yhm!#@CAi>Pb93{1!bMeeoJGHJg!~D_iu|O0A_$#mKUx!H)o{D5v0SyQ
z?|@~<_y{oi_TV?AVj6!?MWy;fnM!_Hu%s#~-o1Uhe|DA+Do`W91B%MZGP1H_^L*f-
zwB1ut1lSJH5e(=*UF!`C4|j2K=`v9uvD|&qH7~E8u;ua)@v9!)lK1F7}EDiwnN5Gm4
zTfLJ&_aI76_4cJ$3aPxGJTV~Zi76@D8ynJYXy?y#dKna-<5*Z*f1q`O6eN839nou2
znUV3lF`(UxzPHE*OgT^s0zc;ppgd5j!lMNdVQSy1rx
z{_NbZU%xgZAbX(k40LpKtgNh!st|vxCa*JFSJ&F|a#om_|DRtqXx5`yK}ku(fD9F_
zL4EZ3{M@}!wcSrCMqNc)dp7QsDxd@*4CUHYW&mBC^ZoBKUKZyZA0Go40fZ?;ExoMF
zHjzaiTy-b*PUD@KnOT+DAc#^R1%gR(+~z}M_fNa80{TlG?e7Dp!=DmPNOH5!qx23q
zTkGkv-1}<-z;YTTe{idMA)!P-@P-;U-NUpjEXhtifk8L`|4oHV?Rq_Z8=G;(L`~o2%i9|+MJQrqY)n&zyP)>3@jK<{&LM?R
z#PfX@WakaN|KK$39i6Z@k8y@&>JAPPOEQ?MExHG2ZZLyP!pmvGYbg{`CT37
z4r39hlpBbybZ6F(VTj)3<}Z1k?x=+G)K!1{yB73hF@q2w2^bCyUDEt>nZtGYzfErO
z!>6CdspI34FX1DZRnGYkh|6n}w|1P-#1H+DLKMKr0ak~M
zqigNak+mXVb(Nkx!A=K>B)Ii+j2`3Ic-8n=rLZ0zrQgtRb9X7J+N{N2v$JJhZc@X1
zxcF7n)DML+An~y>xr~~K<1@<1()p22$u7MEkRnW#F9ni(tT*XJu6Rb11bha;&hC8J
zpFb7Y6oOawnYG-ChK8D-6#uq)#=Bk~tH0ST<=))d0v}MF@XrLUxXQ!HTT*gAm2X9-
z+CG~{4k|3XDQkvuOiBM;i|OLHf!*&e;`WxD;GBikpjrOy+v5|{zctoLa^1!iwDKex
z0hmiI{nd4KWodelJ_#SR)*rtni`|3-lQBr1=5#7S)dLh6C-A}5Szi!)Yf+EGxD$)3dY@?{JTy9=B$Fe)R1g^lgQ
z2V)H~A%$gjlq9sDndzlD7>$@#}^jr6)B;a5?4QFcYa42VvrKHcwZd1-Ok;v
z_R5lyh90bR2)LaV9Ov7+xa>?8lTxkjFFSdirpn3tlot2fO_#kA_h9CE@(~`ckk-iVlHMVwTIWCnh0!{tyu4`ymXDG51
z{o+dXs?BY0Yp>1D^YS>pq2V2fL?bENzD7o_*z@*XxIc;XANE5&5akL~^xU~Vpe
z@vE^GC~OgOkK%XVuU_lVud`b^M2DKf@f%%t7mts3_+55#$4DX~nk~nFe*OC18}VJV
zibZ2}oe+3D4n95_YHD6~$NlZ;tvWlAbGU}}G(x-8y9vw`?W*qc^XmD^wf*Z0YZNLl
zh#99CpuRf0ylb4AQA!96Z9Q7tb?dz3rqU4ZaHr
z3=IyN{``#E+PbS*I#=VcA_k|){9z2A5JASm&}%x?1jA0ukwEZNP54W4ON&ds0wORf
zE-rs)s1|U?@cHSDzg?tlw~Z>TL$x;2V`G)ZAM)S4!4z_$wY<4R3XED^^;`MtT5UCH
ze|fCT&Ao1F`k7W9uC1G5SRW;4V`H`bz}
z`CQhgN@)N`XFqfgEIaZYt%slOTR0y44UfVRAtMrXVb-Z`RixajuP45_ypYKqgZde`
zZ;h3LnGY6wr5IA;*7%2&m0?`*1f~#xQPv~vrdwn03=B$J8|P(2`U9f^1N+DGenTy+tUoK`ws*JV1Ox;?QW1;|r3)nEwa-(2F@5Ox6F{1D1O`=LiEiES#Dv9U
zk+zzI$ny+Uz<7}hx^puCubh)5{*+uO`LYyGTu>;*Zr(MgS&trjp;)sH7#c?$KJ6y#
zya|kU=k8G>?0fC#(eQrxm*K!b7yy}ipDQurzq?ctm);$Q&-@jTkk6kyzTp{NPU4tG
zEOe@@M`F5WjG=-0PgS)Q7Yv-Vxz
zEZ~x-gDGKjiUw29PgD6s4&0N~(Vxjy-C_XogsrX1YJ4_Bncalw(h#~jB5BfUI+)r9jR~9@bvAWow~Gi%H`*dWi6d#GxH5xadr-;lVj(>fiP4m
zCc^&3MSdZDBc5C&dczcM`87j2+C{?9uAe_YtjWH_D{S?ZM|hMRS9zZyvOj;$`h+Ci
z%gQu)PRI=Bz>>?*rfDcBm>3+)MPqDDjEz-92?N2ZC7>c5Rc@Umyku
zCz)*1u0q}z(qF2iX&t+LNf9Z#+qbY_((%Q^@4AAI-O=c1-L~z9(9+UUO24Vq{b0St
zt8i$@RaseLC{32%<4{IrOk_gc8lLR;JEGL@(sCrYDW1~M+2un0T>mz>e@*74iul)n
z02i?2tl*5Ansm``kIxU#-Ojqzm#`t@n{aqf+t48MYaX8Hr3A5M$F>?nsax%#o*t^e
znK>Qb6)aRL2Zwp-NDBW^aRAk~$b^KnYF9g|&?bSgk^Qy?p47BmVV@eGtAzCQK{tfp
z?FKZ(<|ES=?7g!!h#HKUo<4!Kx1TYNXuYVawiaZc^)B1Kj*flJ9xH|O4TfzB<$blF
zKmCb{Z4NJJ7_2Pi2EN5Jn&XN$w+I2Qph3@8Qc{s3avH27Sd(J2Ps%&}nmk|Z$B(<>
zZ}F^{=nIZcccovvAOayptIKw)785a7QQN~aY|$4I5;xb^)Bw?o!pLvd=BpS@jg8;x
zzb>|HKXcvT<#sgjQ!}9*DVc8~;VwD)^Rr4iGTrYoCM1Mv@_Am*@5}vti(keSu6x2%
zeud3?qCU#X%3>%+RkH!ekF~YFt2C`0)f7-{${+zz(#(vOUW21_>*_$#OJT#S?3Bz_
z2AZ|r%ag0ZF@+NYkfC_}ED+#p^Z&NHJX-zXw?-sIO9tWPd3FCFGd&X%78+a*Wov%=
zY_^K$uy|^Mpuz9f<9L7hYNK?mEtrIZn>*|qN#BOGh}UWNc-2J&c_@IhHU}sKV>C}*
zHAJ$~Zt2xU46C<^oE*j*Y6qibKf3J+AxcAywpRR^`r?+rV7j5Pesy>yCzI&6cuG@810rzMs_T|>puDXtRnX&ETyBbu|-+w*YX00IJ!}kO^9Tv(*Q3vkw(ovWd22ys!7cDD@}WUBPcl
zOPN0hG*R$7YtWZiEVaH-Pyirq10M>5OYf=$)rb;v7QdXI(JIkmbDFIz-JaUz;GEb4
zzVSNu&!4h#!+g70yf#~yu$|p7D0Ls2$NFBFevOa6K)~qA^sjz{@YH&mJ0T%Y&CT)6
zn=YuI?dbG0COUeC?Ho{}q7zEqUT*gRz}L}D0RdTt?L-VZ8XP1}z#n?<%+R2t$3lHc
zINj~b;kVCEwgrL%&$V@u2caU%XG=Wt>E3nPRhQON0|qUgX8^HV?Md?!2#346fFdyf
zWW_Gq-LZ6cJd%*muESH??Y%qSsG%sFwvFMXp0=90y2T$x1|N44FiEj~Xx*FZ0g|U0
z&#Cmsk6`$j{BB%wz3t;cz{_oC;4jRqgd|8|*Y#}^MAi<6tMN!r6a@I^z6758l(E78
z8dKtPobI)qg+F5}HtqicV!7X)CFgKm5NXKCh9*WueUCd<3?;M$J=j<$oOlkOtMLMH
zQ|K@`{tsbtt8X}tcefZ)3_|yFZyTflIBW|U!WEmMOW#%+);J8bf6~I8
z4@Vx-ByCYsf4(|B$!%4T_o*`Jy^N$-%>6>B{O0nq@%Gjrk#z}#4waui*#eV{=Y@Bg
z5J37yi_|;j>nMHQtLmz%fZEhtN5YanoTazB`wNd9B3GbG^BFNQ_$V39&*>O6#(8g!
z@Ld#RVNES9=F1W}ImL$H2LqZT4v-$n^wiYn7e{V~^Uc70B4eOr2*l-$mjq`_h_?B{
zY;2a?YFzggmpeX5Lm)*3s&v>yiDxe`dHLraA^)ASP6j;0cQ80T=9(_f)u@>2leFxR
z5f(wguZ<}A`T2~LmXsEOF(sbJm~c=j0Quwf^;~uc283%WR~8f|cTuUZurwY#*nE=*
zl{%xD)>39E+?i#gso-{UuvAogGtWo;3%ue6z%QH
z$7fzg-m(_ZTTTi&VQD06cKY7Q=eW`6SGn1dW2J6LYH*FG{Sa;qKLDZF_n5&$+Hh+A}L486Tw_5(8;%&DqF6`BJ1tkXM
zD{C2uI$eC!m^eSQUr8j{KI%WRwEwa6_CFWp{@WR;*tbAx;Xr;+uAUY#EVgm0gX
zRPsSyryh!|=&@)TF8TLgcpV7
zRYGEi1*rylp{c3vh_1`Au>P6@;`|5-sMqA^sK@Wq4)s|@P)jvJYCS_t>1S?Dkg@1-
zyirm4`vVv08i^l+c>_iule}vsFJ=6XE)_XZwqs-Sqw^q}rED%I8L-nqj<)De+Fu=Ftkcauxl
zv2%NxwXhvDTvkSAs?>0Fa{lJxNX_a9z^ji4XD8^wOnXM@{1k}r=dN9fURK8tXO@&O
znGHQP&*ulq7vBSFE*_o~96VurEmsXQ&&wuD)eRi+0l7g`Dr|}zUVi?oOXSs!4Nx>M
zt}Z#QyV$!5Pq?YE==@R9@g<_7&gm)IW9YpVYCLxTj^;WQEArjMDR_-;0+12SLY
zQ4w>K98414iJ6&W1T4jAL!Jnw56Sa5tM+n#^PnM+S=T*`tsBTuh#kmDA7+$
zP30vVZwY01ojt$kySKu?%Xk@3!@_ws;w5VF$mPWS_VZ?(mz!>#)ym3v$25<>PN`Uc
zRGS=Gn&VzsaFaQ(f%sRb-JiJLDpV@+mc}nwn-0r=w}2508od!qaRLH$lx#e8&jZih
zuM!gzcSU}v8pMe3d#d&QpL(LA;(EKKAMwSvKxoU!&S|w&hELTwTNnYfIunsqwjJN{
z8h@_wrY8;)udEN;R8{lZ?fMQ85nkU$O`NVS%Fpr
z1!QsCj(;zNyy{v!Pd+oezc6h6G~3hj{qpnh#{Arzxv9B|{s-d<_xJ8UytzUov-0y<
z?`pfHv4stTFF#vhZmgRC#gR|-(0pCFlVfFJ73Q~K=_a5hTx#{wZ%fx5T-0RFe97@J
zv`eS{{YB%6L-BMc0#;`;7sof4;^)mElV+&^#l1N14cL1UX*E$WyVkz}=q&2^mc7W)
z$)7;PAj3YwGcrA{_^x4I#01Ah>^qK1oq~{fPeSQ7x2+DWe9sLig5pvVkG4|_DShE>
zVS~$WU*d_s_4O@FO1hwwy~~Gw@9;5v*m|@1bL9|)3a@Z=JW)nQ7KN(2JMWMNXm0|e
zQg@i*Rr9-HS0q}#Y^cFBil6=(nf^t|S&3lnLsGhEsuBTEewUyev4MMCn7Bo&Cpi2*U_
z=oj_tFw$qw{u!Cso~wqq-mzcbJXN$?K_yMm;JX->n;u7y5`9kD`?Fnq4Hcc3=x8Z@
zTu`ZvUrGeIuqaA2R83{cce$)9lKTmSb{obx(E;NS5h>5faqm9%P{Sld=?fA!y1B$G
zbP>Ls(7fSTEK(;0<;U}$E7BkR{r%Rv(icEd&`=XE?|4XX5fe%@yno@XIgKpSpMf_b
zWA+HdSvMoT`utaoywl;4OU32+cK^uL7E2~W0z3S!8!{|o34z_hNw_D+#xA||_1?dS
zZVW>aGY!!=hkrea(n@`Dn{F&X{wVP3h7QTUC|SU-F*8%S$u$kkw_kk&13xM&x!Kvv
zRDYke|J2XRGs~r+NvSIVnZx1lZ$-DnXD6tqkAu704_gikxk%KOB)uU+S1UIAr+
zqi-YC(%lbxhk=+}Cn3Q(SZi7;5V)QNNN;U!(r>`jf#Uc@9D1K_?LM#5sidW4NNDKx
zy)J8`bDHPRLF|)!1AG0}e
zget%ALnD2yWw}?zpcQ{AvC!7`CF*l`xWv}iPDuw5+15x0Q0mF1p%^F_&nfwIl_*6r
zCbzX!moC(_x>nPSJfD}V?|JJToH!k))AP7y+vD5LqWW%`k*oTWBz3esFByD0>u*-!
zcsHO0*DAKHn6tkrrvwb7@PzCw!p@HM&bMSOXc!ku=X-%6Zk-!i)5YFdI^?EYzqS%r
zdJol+R|lKHU>SXs{2wQN%mxJC`8Im@{GkMSc$tRk1OJ6RzX89~op)e>!@}ha+B`0<
zFUH$-gqhSBfVTD4ZZy92X`Od=Y;5Qlz9mkaNtZRS^&FP~gNK+@kGaAg2BxrYnBuat
ziwmRT$;Er*%X(rwQ4dg4hQ`NN+_hsxzcu;rfrr=l<%wuqOpV3zZNrD4poZMB6$?V}
zu$7|=?x|Ym{PSligJ$m;AW_?6v-z^-l#Qydk5Tfd_HmwqFz9|~!++c6@c(k)7(X+y
zJWBp)%Nb)v?>Yu@OPL~Z9L8o`L2da<44I}uhM|wfgP6cM|9!Pvp{W!hgNP-Gf7W-M
zszpJDT!A2+(ZB!<8OQi~P9p*{^eu&-dhEobqc7H<6;k}U)0b~8v12HuPaGqcl(*>l
z=W&8aJtr&LG>?rN171(`hTG}5bYwp9UtJ@>6VJ*{1SFg7?hE0XHB4B@MCbi!>vl0@
zeR6fKk1SrW<3#c+9THvmv+qS-!BUeb{kd-%2;Ti3YdWRlX0h@WL;ceaNNx@*($%)gZe-mAEepLdLnX9ZPH{*%WcQap#o3N-RA)@dg^3Y7e=@neaky3`5~WP
z@odL`?S9P;Ao0aV?zFoMPpO?lyd(*vggw!OAfa$E$fongP)A%GUBHtw1{Rhiwhw9@
zUtCbB)P!gQwPyN1OGT09j~R-@Y>_y8b4qg0m8Xuboc)+kiA85z5!hckHXDLWVWJdM
zoGIGvD6QtAVQTa%iGK+uMt17cA^!05Wr1|=aP0&jOdFLa`uC+c;>l@FHjC5R7Q~v{$oBrj%#cJ1BX^M+eQ{@0=hEa0}xCr%T
zNWc9`H|Af*_&LGs#UW6JblUjIPK>Xpy7@V#BnIpTEG(qD-&gai#Y+*DO2Cy2sNnP)
zt3j`>yo`pjmwa=GF|G0@4wny};V5Cb6ji|I6WFfagl1Y|-_N=hM-
zC17t!ktQ&VLg;DpgJK?Pcn
zVtQ^a3{=43TZe19Y;24uRC+8Mo1>FR$j2E1%DdkpDCGAD+|A9K0-mgZkOmZ2Ey8|2
zqVHvR_Y%+CWc7YyRw>vp*ya%{hQjBD@C4LtAdZt0zK@T7qtj%gzrO+*)?nISDXB~F
zZb6DVzdLB?Hf1yOi|wBbeKRz^FzS
z@=dHaDX93=&(1kDSGvmpJC!XERB2Rz6A$9X{(d&?m!dTxT`@g9j|D0AhDGu{J-N@0
zKPtFUU$PzDav3h20uuXWU`}4Eql-(HKtxPNJO+fs=lmH!n{L6x-jE6{-QwTm{|^D~B3tX+dBiYVC(>^WSF^
zucNtbl2v%T3)$|6D*El=AsQIs_UDC#Kz=7vT!7=Yhv0Hrf7ZBC5Cinx91W}&5(tDm
z04cCVlB+Rb0$@Bu$>ZQ)17Kgu_h*L}I*W_x&cBt}2E17=oZwR~^q`toX@Ue3vkF2x
zrfkoj`Iu14eZM=4laUcWRmiKRT(I>F9l!mHva4-nL%kXisNS|R8v>(uu!akSZ5Kqy*fOV$DM93v9?2pXv
z2YABiEq3AtZx7w=sqn3c>_brBEn)EKrG!uP`ke06gOM5`my8B_kQg9i?%l+CAQ}v~
zb}|9i-o-^3%|t~vLM}F-Z}=jTc+Dr^%2ZLw4hE3Aa1{#VD>yFD(k8FiTv9S>UlGF_zM?mEA6!~t7A!N`jGXz3ndU>n`|0yD$URjs`N^8kfM>Ny^Nl6Ew
z$aRX4&k*=(d_Rv0)ig9{@3*?KE8)M2(#9?Rw-x|stDcd-L1Rv;owuac2NLy=KsNfL
zs;a0!DOkWjIp%?X1L&Feoi77b4j_n~&6RawjWz0{ttJhpySi4Eg&j_rMK|UnT6!$g
z6BALu@--|xb?wu2La{W8AZ)=l#SD#R-!x&5LljWha(*)U6cC(Mv+&oaN*)bBBv=J5r)603%Se)>
zc94={0eJaT@g*K0@CwAH^U!9iEg!&epGrsv`<=kak&uKY(>x!Qnn2!v%U3jfBPk~)
zq``i&DVryk#4au^9dsoCx`Jo+`C(05l)XpSwp&3t+Tqng|(N7P=KqcsR93i!rPDB
zy0o!HdR3}je;>YKN-!~9yWg0~#!JSXSS)--SeV-0K4dkOD2LMEegMRg{hJEPHWoC+
z_XrIrUfMvwswc@!HM*csbq)=|_m>HPGP$7kXF&d98??{30)=YAKLXS>hlbdNDei{*
zM9D=946lIr1niRz?x5f>YahwH*^J18gp)QQ=3G({Pbc$i&86VXf`T&(3vv8`U}qZ~
zU2%BbdE;S0$R{MEHEinsJLt_#%jQt-EBHi3;@w=K$ftk+c`Yp*Si{qf?rvjPV}o;k
z0oV|!{ui;BBZicZoyf~OgF*J}ts%94^3mDe=BmlnG9DdW3MdAQB)+Wz`k&)vr3*5w
zd__vT`1Gpf7kqpd3_R~qWGcUfeBCs)wPm`il~2h7#lLFWhXoB{0iN*tV&utzrMZ>W
zbr-X!k>}Z|gZ+;LKWlSy0=%)bDM>U!(x$bYS!>V|ARH!GSsn!~F)vyf{P~ka-bBd~
z=tHA;;(!+C$vt%)ZML2jU0_^tbx>0N8ww^aMF5sBq>Tg-j(J)W}rY
zT4Te*`I4VzAYWhe^YYCCF-P&jsH#w4p!4+DUm!G<*VKd|bCwq15IlW4nwI6ckE2%F
zGyIZgxRS%*t)w_L9)s_rq%U)-03q_Hf1oN~Uo#V)3UW(pl+#!)DCF&@1@uKlce9^Wr5)udqYTTEE)jj!%X|xZ<(yN9MaW$tk
z7OTDET&sBM#nvhzOmy;57ezY*LwL|VHD)1taSv4M8##wWCG_Agf-jtyli=a9TBwAf
zaZkT~JCwY-?L#B$u5^BnxI67IIbBIvxI}cDaeFhjChPS-Ma^y4`ut=XJ@oL+pa8>)
zVc*zT7)}D-;#=_lZxHN&G;m5=Vb9^z`N;(#(|luNt&58)OO@hBh8?y*1aF3(X_v_c
z_0}0%ne!86!58Sv;1i6@^83~oMy0i*1P%GUt!w^nozZ4bY)V~AGv2<2_Y~Rl^yH+B
zVu*>A<*4Mw7ENFOw$-Pi=9U{Rmv+MtgDO61eq;7lLcNiqZMr(Uuuz_f5`b!V3AbqC-8C7+l^P0oj$e8ce)xKP=leQ&{%9Gcahb{e%D6Q|C{AI
zqrS$f@89jrWt7-<8CVs(YXLhsL&eS>Kds+1bA_|DMi?in1&_yYmHz$gs`Ih>QCa8p>6zsDueLH+XNmx*z|<
z(hp{of2hAJ?MP%?t76*3d|xCmfme*$zE0bSWl~)tTZIL}BO+2b$v}0-4eLgnW`!B`
z6m#|^za4ETEF|Qku{1^?$RD7|$^Q89b~#8+0C-Pm9!@N*M8w3zgYnjyw{Q8EKZu|78jh6eb*=pZmX}RymTe@*lt!^G-Q{zUmWJby)@K?Jow&dpGkt|lT@k}5)^hdg8uxXabj969AEm!w#+1Y{dIO&6b
z(x3N?Ze=CWvyLdTQ^090={5cK?rzKjw`_JTS|90QXqB|96Ng_RZA_8EaDWwjiqfP;hiUd&i)F!kmUA8Nvu9D+r%yNcQtEu
zd%BjcPE{YDJ=U(gp#?nDCo5@F&gJ=XJ
zXcp&CB_c2s_4S1qovmHI^Xx1~vRt9P{U<2i>2Fu(eY4K?_TuvLz4Zm5dyhrb)t%oD
zMNQb75sK8hPS~uBp+=*A@tVI)eC)XPrGbaXc<38aJy^Ivuny}oN?(75=!q8^7Sinl
z`lr>7)ylB7|1kryzd!m7%4p?p!=DI#x6{V1p~)XV{6@4IBW+gZrnABj&{)riEC`;SoYx&;paR;5rh!K`w6l9XCGaW<;#TzGv$zP`Rf)Zj@{nfePnd{
z0Dptty}^L-J#)5E8S6{2Mx`ZjUe3CFUW#oQQYFvUWXf6L;5jSH(A(Qs>)$_l;%gTW
za=E#u3K@rIziV@Tny}F-G#6Uq{If00*<9^SFqyz|$sN<QLRlq&4{~#_a
z`?lQXEOtlG{oH1FMeEoHZ+CU`cl=-hk!BF(PFZ-`Ug9X{2L6Pm+}DfYX7lXr@FG>4
z-Nb=g%Ri4;zkKlrK9b$qq|Ihu!gSrWq1@$&ubWmEAK&^(6c}_LT8{NS1oowBzGl?P
z8LgDmDgnoSdzdfQjkN?m+aL`Nz}o-lZRmR2V8=u=x3?OpB}=Af{GP|E`HG-cp~-=M
zUTsImS4l`g>87W*lzO_bDRJp3)+Z?Q^wASy@2^b(OM~{7Ul6s>(w{#(wjCct#HXqg
zTum+M)>GfA~;r(SHEkDWjoGKc{H`?mG&(saNDLW`+<6>Ayv}Vosb~
zgT7r$4g`S&JRvUKHbX_CK^8Z^56vXl^|5t*jg20Jv?01{_!B$^`xtPq6)|@9Jk{J&
z$DfBr0ERc4tnGO4YP|9n7JF0!p#MV&2Wk!g?5aAqFdq{WWj0p&rFC;8e>|CL;O~~h
zish3#z^WSY#_h)?kx8eV(@73xW(DZGmCa1mPR8J%;tN{WG==D}FjEtg$b=A;@~0+x
zdU*u}iHVTa0hXMx)5|xJv*qR-G;{*eXYQ^cZfZXu-6G`T5Zq$&X=Qc#4BKc#e8yEN+tTTL9WLRZL{)wtGu-?
zPmO2J-%5su68@Q;1<9hE9zKBO?Niv+XjE3%Ty*#UFYvU402`8eEdx(xP(Lou@0w#gc%XT^;Y#v&rCDu*j~KOIahY0H(XNB%If{a
zxuN5j>pMTk+j!ofK9F9t~NM6+?f+%j&R>->2`Zt^1SZx
zBSq^yp2RoC##UA6N-V=01~Q}+WTQ5AdB>5RcGJ_{WIj)wKDdv6JE(n;+4_h++P&eC
zu0ABzq-Gjiie9fRz5e_2*ABG{BSoCh#p1zCL2~?e44Z`#Os3(yRc2xwUURjvoPCD*Vw0=?45klCjBV^Zq^AqtrDs
zgG3v*x5C4VR=o|akPKBi!NdmrsevBKn#eZEYWvlcyh!Xx)rl!lmhhgFt%)L?dOI77
z6;k3_oyC{lBEM&6ZB~Z*0aB;-$eq>;4~LTK_U$cHm*3Q0K*ylWghV!8aDU;^MWw~azt#bQoMm?Iu++w*GO%i!fj~+1q>-*60cXn~`xw(~{T~S%tM2SHTi~@Pf
z{*gSB;n*u1yt9Ap(}nK&&A{9wvgF~X`5hiuhKlMBJ3crOS>=pRYYC2hucHGI
z$8(r~U}{Dl+Z2`;Gap1_e0(=O`71}|g6nKzD=3KD2cN8DcJb-F;Ewc1^mtsHZGyXF
zb(VuHkuiW};}sWoP`+9qiuvgi^2*h+%RU!nb-*d;R2uU2u57Q{W7_4yDF2QwrLa5k
zf-XO2TIKE&y9jCt99LNGtmw4fcz)_8-HCWX-_F1*!$1&~-fe3oE!lyptdtTI%wc_K
zZnd3uc=&)i(LCp#>XCwGX8v6g8}gR@u+f@5!)VDActi0tJ*BaJy*M{P4Z+g6uV
zR=Q95cvA|QjD}|8_ll8|J$z6`9z9bxVmniDqvk2D5yWY;!df}|g3$Qx-OnsbOyqbg
za1BDzXp{8BE7!f9N)?sRdZa+V(+r28p-{hD6Uoei_RrsyfXl6sD`oHB+Q^NDW}$en
z?nZ6BMYJyrCkr%QGV+zdAi0O6m6tz7)}1uhyJi?=WTGj-$;LwQ<;w_>j~`7S35R?c
zxRRkxBG9Z-+%nlfUE_KpkAz2?HGlci=lEFUpS$Iv`}t*geT668U7KeeSRAVO)@u;J
zp1qv*JBMB8(OYD8=3`e(!9-V1rPN;R-6wlg3LJuo8Jr_66rFKTa|$%7^1N>w;T*W{
zD`zJq88(03Sf6p1#3p-4+gw(jMt>xLuaQzUWnr*(u$I}UQDT&~HGY1L*iL&P=_(rJuDOV-bjL220=?MsJ(r>+RE$;e
zt*R1#p8BKJk?#2&+htz0$l~JGj_{fYw(*zPS*ispYOrFzV*4CeLd!?%Y0MYc-pdhK
z7e19qxjJ2*B+0$4tjQPk9Pwhfnkr#ILsk3Tb3`j5ZiUb>Yj+%_fqa(T*V*dr80}p~
zZmjbbi2E$zJUp%3SHm^XnVHE}rf!)c9n8nK6{~uU#=S{D`xhjJw&_*xI&bFBRemo)
z5Q#QCqgF+5>_s!Wb$q7uWB3(EzAewnF0UyNp|7nzFqCNb?V``|zJzP^$(&P=Lh{50
z&4vPVdkMnP@?o^VOB16&fOqP0y(PD8Y}{*gI^r_=>2(B{SQ@X#=F_tRy&r~^Z(2Xx
z*d3`=Ak#6Xxz9da)WAwdt1HQIrPC
zlHp%69cr_uL)y$nz_30E~yaUWUZ(y-inJ~Hin%Y;cFJi8->g&0RLQES-b
z=F&rEPD%38-+Om+(#-3XSW3!yNc?P;Kbt6RW3|n1zpatMqn@7V>$}JAawgU@JQis*
z7D22V_DKWZMHqI-z=9K-D3^9VxN*9h=K3`u;d_GCaiR93Gpx0tpDP~m#v?k$BXlp0
zYBeMzlnKs&OD!(ljg60=KT!6-i!}Ptd&QrBi>iZ%kv0ZMn?-{(<}`2R^JurVF=~~9
zgtQFRGTw(nyt-^_*|mokWiMTSL#nc@szugU^4#N@XFcVU2eALu(UDIkbL4;jUg8y-
z&|a`snu4o<_3}=$s!uQu<15_b<}_D&lRp;Qb9~++(NU0iPWQA^!C})ie1#V{Y^&n!
zRbll_ZTW^7U*ESk(1P|__n7c
zhnU0Rugc11nRsO14G~zoZBO>;_5jqZdH%O`>&=b6V!f0YZb}7(Csue*3yBhYiEwi4
zhKQ$|y1Jr%Bkyq0c%^0-@F6B(;(C}l=ib~>e|Wt5!i!L-p;y0)^x0WQ=FOX`vI({H
z;peUwsMFK$>J?PRLN71QLH_IH&@yg+^v3WG8uYUB0QoyHY^QhGk0qRikugH@R*3jmhHPK2kU3sEz%|{IL%xr9uGBO#rjSkke=Byk=L|F3B8kMhIJJw5imgE{t-D+vY`t*szD@{#w
zs653f)}x=^u@MpaKYrAgde)j=oX-QyxWVlJc8|6+G%_7h~8nw$AyHIL!q<^4U~xECyx@4Wr|K;~A~%W#*S
z1<$1=tRFv6Mp#)MQBh23X`oU193g(Bk{d-6-eEz?4e#ou&Sr#>SCS#oFLVRa3!4UY!;ziwV)(001qbg2kWZap$roP16;$c4E;R{+Q4!%885zxBBoBt
zVA?z|cZ7;9_4F`XMGo69_UJ$zib4A?BbRHa?}g|xF~}pdb`>qELd~aLgt4z*zfLYB
z_{}b&5>TqZeP!+Sbi_j<3NwzR}0
zbZ7BmZL%t_hcaPy_T6Bv8s=V)vxnsC`qK%>i{2-mL{_w$;Wb$pu|8#4@ma6GYMCAS
z=#lln06jW-o*`Dh2fO->y=5X;`Sd>jI=*`K#o4i|Gy;1mTdm|J0o5rxig@@lbi*Dc
ztM~6$R)%YU;|qO)eqCuBGLQIpyBV*@BoUvlK}xbghB6XaHg4OWAFP_A5(unT%_^%{
zs1^w`GZ~#$H$kHT5m-n44-c=v)!f{E52VfX$;dn2&6?|p7vue~^dmHu#jScc=fKOy
z4(c}SwOX|6z8c-$TTXPAj@FF!wh-40nYXVH;$@A0AI6PGY7}U)+uIw!9w!(#{4Pl(
z&<%SbcV6mg*fL(CYBE$u%+40^Dd?7{pr7CHgDkB
zL`6Pb!;6wcV7I2F>A|XqWEKpUOnX!KPDfl^4EapZ&rEf@Eo5di4_GhiSIou=3p53I
z5eQb~v&W1N3b_pgJw5l24j;!0UF7I@AFupjzhZMWxK(lGe7=U%r^|;+t+B|cO=JqE83imjlnF}9ryGLrf>6#F~dyto?-v@tm<=<
z-E`QC0wemvSmAW3r&6->!TXJw#h-1rK8#`ev}>lFy{JViaZE3X82jmv$v
zqnQ~uzrd7HMju{$?wo!%zj~>mFv7d#K5<8xNz)~v93eiuy=g&-*vCRLd*Cv@!hT!5
z8>{6uUTm{$hg7Td=v`c0Z-n(z^vjn@>od%$oVl?vl33o=Q*B%Gl^OR-qTl6S^7t6=
zLXoNIl7?9592SAc90n;ctZ1PYPFq
zu{gJL1eK&UPptCaSb(msXja&z=Rm$Mmym=+j3a*L>d@Zm=vGUhD;b$uJYOKc^#T{_
zNj1yEhsqqI?H^asF{>@sx)l#ts^sYirmFRRI|F}hvZ;ytekMNZ9`cd(!k6ae$MA><
z2ma|%Edr38+JbdcL^NZj_Q=DIcB<*`md*NzdTY=NtL@Y$Q#rcSiRrKkLctn~bV3R=
z83-tmVRnRhR#wFV&igDV;zn+usO>Du#vtq`funeb9oZGAs8rU{-ydP;^+o=|<>}Ev
zmh)&GDCZr_{t)QumNvvXqIW+hvxZq1wxQ4|C*T=Dc|;a*)gsK-7f3|bDJj(XZ_1?Z
z69M)j);VXGPt6TvavvIaoF+
z9wXbkC2%!QZD*>~sEJbp@ReN4V@2XfzcBzy7hh~%PVU$=ll2LqGAhN-L+cFt@4!q&
z$HdfNJ|i*gc=8%mv(a*VQjrQ*y2)99e54OxDNa;8ChX6
zbY+x&lR4;_LM?m1GTzYvDrBnm8-mG{sqbMXcRhBwP3Gs1f73@BAVX>xeWox#0YAMz
z*KV7Zwta)rhvQ8t1M_wJZN?Qk{@2dWy4TkD_q3CT=fnk|bBMGRH1ycn+NQ!fbc<*(
zf_>tnpo)J$?fIbWs$qtQamUzee!DdflhL2X!&c;oeP8o>wJ(a%S1~?n^t9j_Qwcbo
z1BKCxut($Pv8PROD69(IT{a8!nOYwO7ZF9W)68Y?Psx-w$x^&JjQsTnI0&{DvnFUTw`t_L$Xa75A*YK7G
z8r(@J`IyD&QWeymJ%cU8MWsDwMDL=b<8ZiTME&_i~
zr=@Lquwk-dV|T1bsMP2$pnB}#tYfs8{e}_|^i90DvtEgsM?J--jz{)78NWkI{#*a)
zxa~QK$!z5s9@Ara6SCJYn6UDPFL}Q%hjGA37ZI12x763afqH&n^;=|jBu}ibRj>ZE
zxZpU(#2j=MBLWQBfw-`yasTeCSIhLzrSHui
ztfLCb&;WQNDK2h0VfXd86Qsz(LI)tqT5hU-kI-UqTIBY3sEqovuc@i->zen%BQk5E
z82~(Z&`QnA^I`Uo{VQwpXa&glhphy9=5=?y@*ybDM0^oua@ki)0X8&5#S2rPkHC5X
z!_PlxVV36I*|DviRk5?RWFWa+ozqog_QsJsUjVIr@t3N$qqII)4_h9LM2@|#tU;Kt
zqv0;=GT5W|(hE1W4ss(T^;UG|$KS!3M>e4u#ImkHa~t#}nY}McKE7ZH?`UdDQ+vb-
z0h$g26O*TAlZzoC6POrp%%_~@V$CDgr(HiOpJ)il!=24FX*{%}VBc$qn!Y3rmWW#(KyeUer_yc3rv`W+^Lnx(hporv8%~fS*ke?c#5pTD
zcR$pQPfHv3x}IF>SvZ>XY$_lCPnkve?AR>P)rYb@>5=hJA*iu!dgIn$V7%FwcBOsd
zum-hTQaomhE%gF`owmyZLA{_ng{&|)Gd0bR-!ntnJYZn>+szDTvj>|}sNDE&n=(Kd
z;Zrz$nBoI1g^%3EV>d-Isi(R1IyfO1Q-Am!F-6KoLfDrL2qI8=6!qCniocq~1$WGQ
zdos1qEfn#s?PXvLuj6*F`>Ca;U1kR^K0e4g+1r#ZPa%X6CIP{+l@`C#Ats-tbV2=y
zXkuI~CW_RE2`%^L|W@9vB}kl_7dRFUI_&2H`(#<*8X
z4GqETdCjz&lUhQaMXK&i$yQ8}*MpOYOg_9!
zZR0dC#C6JjvupSh(OUdbxpHj+F~vIuSE^e&cP-~7`O
zC8)HR@Op(Sjw@eHOz-!;kLuDzKJBm;40ybut8RKy8DOsEi>0bm6q_!gc4^5MkpH-lX_X?71dOi+aij!E^ZT;vQf5MH1Q6m-Y
zgkq(ZYI_;e_eKz4wvdSA&>2b@(PR3{g-1lRL(eWPf*>!%b`FMs^K8I?f+I|r7$1d`
z5o8OBCqO}ccEc#VD;4sXPn%mP;Yr;Oh7
z<7RKXlj50Rq3$6$+RW7ERIeN7-LxsRuo?Gg=+=5-0K&1QiWb!NERM~Y=nWTcadzO^
zl_0QMkBFUJ#u`A$Xq0;rDiTXQ>U^
zLY`dE6q%EFaoFC3Podh9Z
z@$mwIBpJ>@iBAC0K;=h2)zkB1ra4Gjycvzsc;8Rfw7riHAaVFL!5jD&SVj2wpD{`FNzQF9EgcXo?_>ZqI?D
z$&+DX>?^-Bf;yz6)XR6HcZc!lQW`j$Dfe2(<7;>z3|%>7qsU^JV{1Cwrm|gIU_reMw8(`w}iBODZINHV*|kpaUT5GBc9uLzv!E7ZIeL4rK11VmcxWT%-u{0?G@Pn1Kz2ST
zfPV)S;P*V|7M5z@@;?A_wBiWgd1sl5*ZjxWl~_X(({
z>Ad(INW)@di3XUyU0evgJXJQvGGMJLEc74*h>JYWujb~`%*;%XZ7nPW$W!fyYS7_S
zV!6_s@1dwrv$6~)Evot^B?`gV4AXquNUD7wv$b6>13bibhz+y=cr)(@tr
z2GZR$6;0n8kIpaTt$KPmsD&bcO3-UJJMY?3yM|p_<%#>GPj|ec;nm~q27*9?jZIu{`!fjZ&8sSbWUhkO3zRB
zYm=FB_kmgqs>?Cq;ZFhf;`Q}ea#PB8+WPpXMd}ubySqa$+c@%?ja14jihQoAXNjO1HqtOMCJEc_eL
z()b=JTK#RgZEY>Zo}Qmxsn`xMf$(sK@j3&4aNel21g!ETlWYvl9STZ;av>q1-r=+n
z0D{Fh@TyX4S32~^NUOWOt)0Wd#B6Hx;Lv#~t3o~d;7{ik=+}epF)5eBH&0K+tXJi3
zO*lnknf4p#XD6=DP8e2vw*ID%jg3(WR15JP%SuV*lnYYk=6#qJ@&|ALr@z}&q4qp*
zludI@U0wK|&s68<)uP>9T$(M_-z$fPU&o(ldGHE_v~}KFyZBMXC=?bmbRea
zd>k*qWTAU>_6q|sRl|B2%n5U+kEEPhRou_fa2O8w?*2pS~FQwwHi1-d3HKtvV^zqdi}IAxan6|
zh8_okS=B~5V>N?tAIYq_Nc)I%zoM*bnBYN>JBhYpBJ>ThVEkcszwFCz0lD_g#8=YY
z@XzqtgNNMQ{c{vJ9x_0B-B;?pJu*6)LE*7eX?+Vwt{yU?m+AQb+da&`+n#qu4676m
zo=0->_iU0rJ0fvO^`i+8ZW~Cf+<(aLQy_-8qU7vE>+^$p{$knBA%LS{$DJ?YoMUrO
zTs|Sr?n?)27yB^D;Ek;tMPy;D5G?y~2bj6n{%jWv;U{s6xGF^DG;P?%gvT=y#VW7L
z=o>Li)NENXq(KHmQ!}Eg{MzdDx^y0P6CKXe)lc&BQLH5`rYs+MYGSfKeV}wJ7)<(f
zx2h~fIN0^CKeBm1(rl%m+LkBY?u#S~v&y+GT}BA!s^IFDLvzXPDq9{;JHlfe>v>M<
z`O%92TOqnkvzV(~6+MzllE|m8XH}j%rPvL-dVWj`|DY{)DjGegPGqyr%dM(FUd3zK
zXcOV4Q#V0G7Zw`ZAZ3V^%*;S{*flbUecHuJtZNZc3
znF?EZck!fBKU}mJ6E-QPH{T|U&TEaa5VQa_eb$irD$eH
zhF`?fs}RnE?>e)x;<6X0Cbd=$(TxNJv=O?#8q;j|BJ`oL;|PeyrQvf`AU+|DAv7S`
zt5BMl854P?FS8r(V~Q|wtV3fnm>18zOUANc+isHol-F>5-7M`)xheK_bte%TxEMbB
z?@n48sQtO#ZBOuk&_l2JJ!X207Oy|0Tzir!UK6%A^v*jEAM|BVqGPj=-Irrgc67X2
zs)*JoL?8Mg{0qvUObH7>OFTX_v^UE`(O)|clQH8QD&(@J?wt?@RHR*uD9ehoZa0q
zHwIM?;}=?5UQIP)fQ2Fc7NN3w(uZCcsPyL3zaL$m2QG_36>OlbEeVRur%f=C_}Ukt
zyIo5%l}zJeWsQVQ|Dl+%@wFQp?=k=TIPq-a(50oLBjECHzF}eP*Yd~FgfRPfyhRv7
z=K4V7_(J^EtENV7ivOHEkS%{7E-WzqQ+^Oz_4g+YOL}!0j-;hXa6x*w%WSrylSzxccizs>g|kA#w-W~aw@*Eh9HUZ
zIo9W2Hx6HZyfc*N+&j!YHar||_jTpQN$9f)Vw@N6-_xRq(KdrYPoX~u?Fv*ns4Bz%
z&@O&1?rizCU%RaBqSqfFQ}WyHnxJstzPlr8v9
z9P;02qG!?=9Mwe60pE
zFvI=*8;_K`C_f{+#@*5&WtLfW(BV=n;o3Az|izt95d@mD?Q%0G6BO=!(a$NeDfyF9e;!F
zm=_3KB~lhLo2D=(*^xYT&=BGUSsWfyWM2?wBOi#xLrKEQ%JwP&Lq3`>jkuw74ZhtD
z4a5tgI$>cDV!aQ3B1lWL*U7m7AUYB_A-?ojiEAkV{%<;6JKo%!3QiZm{kgn#43q+Z
zwRK5JXnI}vFszipmIXp}ppShFz84~*qWJjuMobuep@7i-=LqzGnmX~B=e6N1MPEDG
zK0LT^Iq)N9+)jYb
z`w5CyU3~%!d~_aDR4xLnkx3@*t2ljfax!H3QYo>rudt9kxvXp$&zF`nPQd%iQ7wR^
zUO&t?5}>bWySvm651zw`F;i-!i1bUsfnMkdW^ZtL$N_-Ci>QbQsAEeaZVG$9c=?j*
ziN!rSx)ku|0I!FZ{#(Q7EM%2y;8p-b1SEvzz>WnBe30K?;3ECd1ZUw};PVVGAU_h|
z;e8m%B||+=63Eoeyz{fE3CtPxz!U~NCu~v3VR%$KS`T*#X=!ENBJke=0~ZU6LXh#H
z#3_XVISis&E$z^xiGry0br5-M3Zg`S81lf4tHW==wE>KNptF8YNg0-2o11%%AeQr#
z^jTSgUqI1CKC;1WRj5;yr0#JP77H@b$H!ETVi|TyqfyOhS>ia8^~gHOO@K
zfh|!ZOr;)Z0s;caWDf*~W~~FbD!_QRUm41hj^+9I@nb0wIz@6Q`gz(~UI}~xJqf(H
z^Tg@DQ{vbRLkoewF^pTaJ+V44%=suRj36!~VIfO$A>q&fh1!u2Ix+BCUSC{X+}JSJ
z)rDtW@`3X;yx=Y;$Hy??@!Y?Ujd(t*1M8mPrNj4%{cG^)hW0-?I#PT6y7r{h!@~ni
z72s`u8}^&fgakpe$tPJ#o0KMQlr7sv$JJyj6ewq2{HjE8_?T~C6a#i3dZx2H-_2I--l?4U~#dS$Gp6}3>M^Q
zGta>zfT&91`7t~R
zvCG?nW6tj_hf7czWQ0Q6!0HCVn7`7qqKnI_%KFmNlO_EmM+ESKRlh2}(YPn;5|WV1
zmk@Qz!octXv=MZR!E`%K!4U-fkz^lD3d8X>v!DX4!CVKLM@&r2+1Z&YIPx&}aHQ(P
z#m5_^DIc$;Ht^IDJ>5O2lEDks)zwW+0~sj>aNUlwX)9a05Q_Zltvk?3fZj0^8&&U(85nj)S3n~?zP;wb=pn?1>
z5S4TMalvDY97EVcjP(2?JCF%Zw5Q*Tw7Gk`H&`tmo6ht|$GT5lOfXYWsEf1#DTYJy
z%UvO}FHRA+S>5Xa*I!?_=@(Z2xR3YsX(##K?irSy`B>#j*K!VRNTz!5zO4Yif7-9!
z#MYRY7+X9Xs?%TMAJNP`8aWR6afPErlB2ktw?BG1Bg^+r^pOV;r6szhwRPkIf+s*4
z0#0QR`Oz@w0n$g4DW(UubGE7W?oHIkdwqo{?*0bvVaG7;U<2h$y7bBIPTr~-XSO%|
z8$f;2>*Ne?cRSWz2D04zoW5){A0GpEG#G4j#39$I<`*DgU$W?OwwM@s&F996(>9yE
zeQ@GJFW=e8)tF6DMFV
z$rzS3s9C@5YhL3);FhSxRExJvRBf|Wjdsxk@!|sYhEvZHG($2$dJH+cErmHCuncCn
zj;2`$^JXL>+>fECoe}xwf2QIH2a-P8FD0#u1+7tVIndLj>PxHcwOKC
z;=p|73Bme#`9W%UI5!P_V}l|{9{#3lsJp!({ol;I0NL1mYNGRoT7!JPvQ^C*yCT(I
zH`rj1a3p}(Qnk%fmd1v`x)b~2l9HkLGq-;UftiB~|F8kK=Qp?B7q#rCW4iFc;Y*J<
zr2u4x~$g2oWDO-wxa$Uhz(5uwr(B|AQnci_vL1F<;oqZ9@%1%Fwerg~wKktYk^a&uqVMk>Y7(@nLste^Eh-Fo`i?pvO<&U%HcQV~{H*+UrU3RyPO
z!842uO!Z|e2MxM=BNoACG%K7F6$FoYcpP_LLp)#oSF4bag+=e6fjerfYHMg&$Nc5-
z-=E`{SO<57EDcvmi|lpGynSvE`8~hlAzjD`^&BEz`eE4zkdgxS=Qf8~bar+(fK=2z
zZHIQ*4_$PAI2J;Pb;$^S%3R&y#~ijp=J0hm4Qt8!J-tVNauj9&z!BW
zQHy_S{T>*fVAm++rdH$FSs*We^Y?Et*!S1i<7O=}*Mb3d!_Siwdnc-c$6PC2og|h)
zH)J;Qj~g_LQ#=(18QD90vR=^2FUm~K@Q+kORNT+6-~csK2CI#o=^0S^N_jsxkEehG
zE)ar=ZfTCco(Lin$Z~JsWI2w%l4;!KPI(OjE!^7h#rrT})>ZQ3QqNP@tgnhpoMhzW
z13e;Y6;!AioAsx5U2~G0znvU>;{6MsQ78s3U>uUbKA=%-90o=#BC0crGE-S=@o$2ap~JS7m-`vl%BL`*9rjX}-Z-RJ
z1;Ty{{RegQjXzt_r?~^)T^#eDk2u$+2)FggT$%Khyq3wrmBn)I_RLJ*NE^bq5iB8z
zi5Ig!JmJVWcte4~0i&hwv-w|ot1aTe9*p6uc7bS0x03}GE4&{9FKTIdFwQmPWN~*x
z0`wci-f8`B`SI}P2La%~2r-<=kCN9sBmqbRyHD6YSztkSoy8BRooK$FF%0o8-+>TN
z`}`z-1Fi=S%*{#(At5(W*#G{pgZ2LXBuaD-1Wb@}piS3%879I(TZbwZW?&BQxU^9N
zB{9nX?5>38uX~F?PQ4Nnvh?9hVeN)uXtOsiFMBJvwWO>p4V=0Dy%tg@a
z#Fr9M!?|i<|MM2A1)31E`JcC7`28scEe3*v9+E$=a-%OB^bh7TP)gpPxXbOx35V`EGGAJshWR?Q8cKZOR7(vNEH<5i&eJD0f{)sz0
z3gAdt?|L!-p^&T4=$Qc^%LZ1DH$?!8k(--q*or=jfsTTGFvE+gFW)g7wG+OlzNU#Z
znq6z9qHNKcd;38pWAX2*V`4R%&(d{ApB5KUqK|%QREZbM%V;(PKz*F8ELCN&Jitwu
zVx3#PuKUZX{wwD#+}mk5$g1g|_iszS$P&@~PPnoY@ao7m=JOOeDFwL>Jwf(OD7`^i
z?N?1c+7MRK)8O;+Qzs+12G9I^m(7qamSYX43~J2I-m8
zRPyim=+h$ASU3?6IT?6zYzvkD?pIndSy01*X;M5W*>tBNd)d@L*GL%QuRlk>uq`1`
z5wd?7VS_0FEkit1ua!5w(siNP!UW)Yc@I`?_xmY|f=AYSj15w>U`BK2@NXmvz@Z`^u&o^Y=
z)B3AM?{&nzd`4yT;2b5#0GHLUVCVPeB;N9#yzu9-f9&SKU2#uL27&FdV0l46>ap_;
z$BLJwvt=p{Ld`{%6_pN8E{9b&oU4{KQw!G{Y8mcq;f$V6WgjFW-ZDic>&}3J=N#B(MC)7|0F|j_iP~D$?t;pRg
z-{0g*=iDcy@%p_}qM*kZ!Dr2h??C|LX{r3DhIf%@_$f_dAQEm6C7W|zK9av2s!GDK
z5h8v|kRVzN8xRN}w`$j$uh`waM)#e^oAT6bf9UV~BDE}ya?94{t}ZV-(`#>{CbViS
zD_K5eZF!&mlFC>dZ^^@olcY1PZ4cMpHdA|b>0ETVzbt3`
zqK^q#Tsw`lH(ogB8sd7IeE}25y?6HX6kYCKm2~p1wg0+l>nINiYAm+$dRQc5~ndfM-dz`%K1mdyEkmgn>Q_Y6R{*X%oHw@F;=
z>9i?Ril=i-?K1HBxp|Me!==Mp&Nunokq+IhUCj&{x7wkgz4XhQe4*V{ckUNVd3HL*
za_3$5o4|5oL8ZkW?gnn1H(y_woBMQ61@26IIxpUzORmw5Ef1K33S16vTNd_h&zb3#
zplvqO%_slzEqwH_mfM00Sewo%dK*%9$$RP6*^+M+W{KAuYo;_ccdGJvv7S4$+?T;G
z^X}C<$Ks~`3TFdKheR!^_;mdAx#nEp+-)v!bn3wZb@R{p?$0D{&zy1V<+sF>UaelA
zl)N?uN+rnDUbn{wzIj4MZ^8Qgywsm&7L70{hGTav||2|RRMFtgXV-!`ew4)
zw|sS5u5adX#tUq~qIY`g@~OvS&eY0HUZ1XDu*oc`!o^?Xf^ov*59@>0>F7>8?Jc6U
z)W}*hK1b_AR_3J>;;B4WfpeJ)qCBcqeQPz(`snJOnELt09-GddvPJa|KT0;rDVvE(
z%>1-g#`Rn&?@=|YZIS#RUIKF_Xer`hRUidheh6>zptnvx*w;x2A8FXI!5FBT!PC{x
JWt~$(69B$Xs3ZUY
literal 0
HcmV?d00001
diff --git a/docs/evaluation/tutorials/static/chinook-diagram.png b/docs/evaluation/tutorials/static/chinook-diagram.png
new file mode 100644
index 0000000000000000000000000000000000000000..fca0ec6507e918171107d22b72f08c9a1fc2e654
GIT binary patch
literal 208720
zcma&N1y~$klQ)WmLBgO3f#4oAI1C=#1_%~B3GVI|0>RxKf;+(-LI@Dt-CYNFy-oh_
zzPsQ3_U_(!o}NC_eYC1hovL5e1Svu#G0+In;Najeq@~2(!@(g3!oeX~pdbTJO5^b2
z;ozPNn2L%jN{fn;DcV^ZnOYda!AS+hse?W!_qE@0mJ3!TJHX8omk=pIJqE!mY+|y?^u+p6{kc~XwIV^Ee7-Hb|9cI|Ps9Sn
zHu5bkn|6DCF8bGLtd197q4?`nYvhx@F04W|w&p?{IW+StG`|hOC1e@`>B0YkM@NE#ME;CN&ilehw!Lk>@B6qeAo2zAHyPrQkMCCk
z!YHae>|gpsbV%OseuG>)*6s{jJQU%s%R5T$w&6w%Tb@;7&n&a|*;~BWiw@X67**M7WDO
z7KH9CBmabV_OVT@X9?X$(}<_AHdWmgGJUi}7ShJ}MmiG|8tvHQWss5~Su|VN2riZ~
zzt{I%Z^*-#O5vP8wfC6BoY-o&pB_q|{=5%2>7Vv%P+zh_Lbx0HQT_v;lU#6s`V{WW
zE@ALXhCbTYyC@IX*4w=@3`tumd*gd&;)@AG8O)v%h?96D&nw6%*)sktMZGTAhtUntZVuha~YJj_rZ-XA`vw#ZgL(XV`jvCy)F`K6F!K3*i*q=mF|yhZN^
zb!Os~z?q9Yn?|brn)VTM8k4YH$p$U;8GgH=4e|W9;{4amaQDl^SD|DmsNX3##7J31
z7_r@d6MP~Qlp@0k|4q>&MIY^hf{n_CuJI+7qBs)G0R6r1z9^V7Kk`(H*6y`y;2A}E
zlmeZ9QKn`Aw%NDvcLa2}7@}()@TYrW%2Bu!)h?i>!7_8Jj$LsWZ+myJrhGDz8Pg4nGjPKmC}>&XJLH^44EIC4GjnSHYc4fXqbkIzGe!Ip4MlN97)9zTdPPwaj1!4w
zG2>_xz2o2lo%egnw8dJ5RpY$7_Tvl_n8iyfq{Eq}p_755e&b8bS3(>Z_ksTUN7ITQOzD*SyqXkS$g*k5CIxYpczx4ROl4aDd6c
z^k;P-iQ#czEm}$AhC#2i*N_Z{Fo&c@j|J*R$3}IBrM)3LR^xb2BOovGU#g(=MO
zp-H{Mr2_9IIPM?pfvkx}KSq{AMV3%Xv2`iyDP<(~No~8aIIG!1Ia5vRCw?s0{QNUb
zH!WpQy|`Vilu;30kzpHuOt`kz)k4F8ON#5$Ll}cgGqf}7KqC;K3$@O@Od(A6Pn40X
zOLR)~;;Q13eapaB%-v)SWi?LBPbp`$W<9KXTM_sp(NM|UwmaIQ=SP3Fw(*S7^ig9^=@I3u!@LGR*E{QcpJb+oPCzIaO|T
zVD-5UQm5;TeyMUvf0^=}NrFTF%~~p`y<7B
z(R;tPBBEuWnRijExyfT&uuw3?v(ZD|lecxEHR7)S-o+os9J*2Nx)5Sy?9vZ|Y7nDe%h6-}%nTTOByqBcC8(38sl
z6)v$d!6}E6!+yzhEsy5>dcBVA;p+WbLidvV59vs%YN{e?s8orxM+RPGHGQG^e1UCM
zw~ic_)JrPv+{pJh%C-5=LotckoIEa(rcgU^t>miHc|2FIrIBOoXoyovxRThfh3UyT
z-#Du8;zRNBli4G?pv5wKTb1>D^(-!4&794;t=!%(9yJ=BZ;H;+pC5)!5VRzKZj0O%_cZyWW;wv4>9@p5$oeNA1rfvc0Mno`u-sH4bLygChgDZJ!4R4+ns-Xj
z(E6Z7k|D(Wp(LOxsj9ADWHV>LG|61hd1d~BYj_tySgi!n$qq^Gg$
zt{NaMd4cnH{w#rsO?MhGFVl-o@bKO{$%o$V{I*jEHd>yE(MY1+6s4Zfi
zlbP52alTXg+tRX))vI{M>PL&i?G*7ZVjVtx=i15osquZb`I_dJrE%jI2kAdc=Sea|n38IFrP_I{9Jk|sqeX@WLod>LdCuz%?fMqR)}VUkVWu!1
zT&K*(wl|zOHpDgVvK}&7EYau`uTa@
z+S98SOAPsC$l3z#VV2thwdJAvrPp=P}-TmCDFDDKY
z1m1cm-Uq-)&vmSHx=E1vExhqAw-59W*6!Dec(9xqNV)k(@Y&nx;WGy~-9xx9$*z!9?>
z)nmXW3~pr#u~!5?+8<3-&vP7E%e8LZksoFZ2^Z}mcnLnD(^qKp6!N37!z{8Bl@;E6u03kBKVDh}oX6d&Xj$waN~49U2d-!i|Y5JV#*BjdL-FoL`nllWJ2
z;Fkb}iGza;goVY~*_qjyo!Q#Xn1z*>mzU)&8w(p76HtT6-qp(ClM9oTJ>@^0{6{}x
zhW7e)rZx_y)>dRs{eJpv?dTvtLGd)uzd!%X)6m8AKS#2%|JSmB1+qN7VPR!{%kuBO
zfu{UVPa%q?E`}BwVy2b=&44ikSy|cH`2RNezh3?4kpHFWhyT))i~Ij*`d?oCzniMs
z8`_ClTLObR2>vI-{x$A@efh73{47sv|1Y%ohtPkY0+bd+<7fFd*96frIg3dFMiQBd
zDX0KNH+w%>_JCLh-79kk&SU$w@IMc_HD=svDSo6D(;XMU9lrLYaswe2QWd2TBzt{vG~P=NLA
zl^l|Yh;U+C@hzXSth(c6|5^W(!0F)W*_6P7YW|>WajR?bu0xuc(vhGaCKeomFtAbJ
z$>5N_!2N&z5bhuwMgV8|7n;99`=9Fly9ZLAmwoZ2KhY>SkpJhN|J8y63&949Q_-+P
z<^NZ86eO(AVA%gY@2B1|{NaeiFw|Dw;{UJ9e5#JZ8o>2G(&DL7A=&Wh$)$1>A~`_3
z-;_snj|-fFN@8d{ffPU6DgU=6NQDD4>Zy1qL#Fr!;RRnH+G75D>PXW6b$Y^6dP7Gd
zuN?Q0;Sf>aeCWw-9uWIHd&bIlw3Se&eG4eAA_@MxPJ}rS_?2F!p(*$Xdtnzi*wMGe
zi`oN|v_N@#*e+rq3cCCv3_;1+B-5<5c&3Jxt4te-@ZWKn%n$Az%#W-+3?iC2dc@_3
z4C;qRz=(i5B>@P$ga;@50{bIKDVHczAh76zG#18x*9RP~p$X!7u&`!~@_8`kJ_15m
zFq})dk~xVJJHq4L7J{4SZb-G+Mx96T`e~Qt+-QXR9k*re&dbSWQ`@)i`ZZQM<^Nmq
z?vQ;^yQBhUR+Ie=Q^P0eczhey@dw
zei{x(lXjx(JN(45O>Mtf5v>Alq|xNmB5v)Em{=wbNLels#A58mgH*OEw_}CMv^XFl
zu?Y#n69S*h&1PA9qgK(wfpqC&_0rWj>J0AbqqYLi0DObPGWq7a;Z@{+
z?MI(3;V%jI-2OE`*oIJ42$gwWzS~dLV0|C!-JG+qWnE1mc!?)vWD=zB!po0NTqcTHy%yvr_n5rqC@f?Um>st
zZsPk>J-?GG--6L&HX<={6I(#k;Gak?tWSnDVQT#x>7(H!wp4LPiX-25QecMX5yE&|
zCt@0~c@nn&;NaJL@X25s@HZd#K?0P&I=S_Ph8vZ
zCAS1-qyi7`mr{sOfc3OoQSzV7VRcEbi)QT&8J%6)uN#WE=+KeAe0kj69K>W{mG`*F&A=jhj
zlPi{Q1&7pcztjq6&Z2y|EDC$OZcN}q2O62JG2bpOj~^b35R!8aCl=k(wL!hV^l7;MeDSPw9j#T2iL^wRu@V6u13sPc7mgFgOAq*`NyLW2xilI4i4+lm
zyCtOixQz3}CM;io7r#XC5J^Z9z6p=7UDYW$D19O<6#*PS}z4G39Q?=&ydW2Z0mOYZEGiu
zJ71}eNUVh#ePE*&%W*a>o+CKg|#4K;mM2(&rWv9X7zo?s%}8Hz@QwKMDDL2xN-^=4_BhpNRs<53ev8>elCHN?>|6NqPx+Xa#|(e_f%2U9jUX
zK%4p80Iv_@2sLYd)C!xU+og5vYgCy`?rV%!h)V7bSLmbgczOLorH$lS?M^Nq-M9tk
zY_;A)V`}uW}>?`!>;nP+=99ox;&x8xkTGAGbp44)_Ulp)bVh1U;
z-II4k$`4-@*=f!XN6(%(p~!}y)I0#rdd~-=W*CY6uAC303crH7
zH&B!5C8K=Q^W_-vV+3eJ2=7CciVjkutoECodBw>a*LlQMBZ^E74|IW!6n&&y=vJap
z6+h`wmzM~803GDt9=1h<^BrtC%vW^3I6QqOR6Z^u=D=~k6UIQyU?Z5@gnXGT)YcLk
zVhQOG!P`(_|5?;j&J2FJwq3~b5SN~Ig;d)1fLyIUMLK3)-Ym6O)C-67dr8Em_ZvS%U^pxF*|5iPJ@^+By?TxWSP5^jq()7<4dUPlTW^4+
zAaWo93;7~!it@!?wxf7-{3c~=A|t0*Wyx&aSsofxS%kfaymo3fPGNxHb4RhfDsW8k
zc`tsQ*xdABzTs>ofRu*IER`
zp2T4}>y(~KhI^J87IrTFXD~ycfhTQ?o$XP_<>R-5hBYx?;UXZPFM}@qc+(xjbm_pgk`;29kSL>3^@pl!}C{!93
zm>?x;lqqXn^#JvnSGR-(Cd-6?C`w2Ik{aJ=R+s$U=|qqK352bLr*6#JET8=-@zaY;
z(Hf=RrF`#TZgjOXU;Re%KKu?Q
z<{R^A!ORuCxyYB~&mRd+dJ|te>kOR(=>1dv`z@%`(iksbG
zs*@&Dk#G1DmjS3HBYMlb4Xxl*K#M%>ub&g9_>-*G9U>~Ej%l?oi~^=;NN)+J725@#
z6VGc9K}v-+Q7vc8Wo2qEuS18brjRqSu}q*y+Rd~?VyBw}`PZjgBkFpd
z8^lH?Ty>MLt?J1w*NOQK>a)E=+pPJ+NQGL>+_COU9&OKlQ(khvF|hwy5FR~vR`p=n
zd}hs&8{qp#C+7U|_xkensZbtYclh_D!)ECv&&#wnyY23U%iKfj8Vt{i9giWAGsXqi
zy;rl%3-MG3A#vg<){|vU;4Dp;h+k0NjrAQ0xALRob95g#tO6D!?Y@U^am%oqqP^-;
zR=(yUkA3sZEFbf~-2U<<#Qxcm5$iaZNbK6Wfw^qZ%q8WH5{wRrxgxgBak>pw%k4oE
zhkjR+kAA)=h*N9*obRw%-n#9c%Mmsk*XRIyC#`41=!}`A@2@L&xM-uw>XAxd1jS&P
zsvRre-fEf`uM@m0r+KXn&QjPZSJ#tUqm^UmwVX3mfL!Ib<*Y%srM2FNU?CUdH_K77
zv;@%%L>9+{6S#3=+^rxw{F!ma8!NsM56GazVI%cCc1=E+Co-JCbuG>d|GtO_8pRg6
z{-k7e_0BOgC_nGC-@~f=9n2i%?!lv5c=o8KSdFmvv=boFlhf7=I^_?AQduWKPow0Jix$3y|1dSG{GMMJ;I5EtK
zsF)^BCI6*yFQFQVT}s$eo_+FPapJnS!F*DC!%`^q5Ez25+a|1F&~Cg+!tAwet%R-M
z1!tfr<`BRS6EP(1)D)N^pP778jKHQrWOD6w`9!3U*hNs`=2T@PVWPGU?1<}ChI*YK
zIWfvQh7a9h3co8`(At-D&?r2JC~~^GXGkG$tHlF!DVXSXd7$Hf<+wi_1##Ga70vEG
z$(YRP{xWWl3}SjlWoAEBk*-hiVZ*>bNx8x3@sxHHJ_G!u=%;fe6Wvt;-b-m5O3XpK
zhbQXvM2Iz|Xrp)qRZn-wVP~Gv{T@Z9bWlts{~uc?xX$lGl*`s`k?6UcT>pK_3?}q#
zB)O$PA>ycPd!qayRRjJN^z5bP_#p4ofvM_6w>@Qd7QgxV%cX2-1!!irmluxZ$btLW
zVhKUD5sc8RsV9#*Ah9^s*PP@tbD?wHD4O)cke;dZpCOdQu#CfEW4{!ok)NDTR$!(p
zwhhxQzGj7Q!T9ixiFwTS@c0?t7lm20rTx(SS&eWvBbGMS!@ra^>F0tk21p$!@w2mG
z23(UpYj9%8F;&EyvZk4lh^8>g833(e9$Po_a_22xu`~_r4dXhab^CK{~p*gfwA&l|_yh2pw1^Y|i%dlX7j$K9&vK6lAGZvpWVi`j;w*J?Pr
zQ4Mss#)O;4^n|TKC!CZTgVD{i^?MlnT)xE$9>g2&Tx!JMmdvL1Ggm_5=c!{af3JU3
zmEokYQX$-Y0j8+&x6#2ya5cYsK@igaS202UDyBSO#VYK@4WwV?@zp`^VRXsY{rqOL
z1;%-)5)TKfz+N8yQ;JZF2-rT~fW2HrY$jPl=f(D?>pP;8#~<{NQxFT|AopU*h{)9|
zR$diplN1HrmVmAZ_pgFU=TKJuh!51`HA@5zE7YKy71wz%#aMWKU5@JW6GuxsPQJjR
z7mInLJ%t|M(Jh}SDoPf7IbAVUBdac?JumUvQtq{NbA|HyXs)d3Ol2;I=Z!=1!{*s)
zFj@SX9Y^j884kab(PLk*M~A-T1VQ^9*WNB-Dwu*ZHTkVd36!w89A)j|O^d^l^-i?8
zo>iqDM7E#*mn%nQ(AM}#3Y#07u{)dAt)69S|B%>XRKj2$I6qxx!MjqQwIuCLjQaTx
z4_B^+@l7%cmRQ{(F_*VqS>7K37sqoNV~XQjGkXP2ahbu=I~Ep#uF*V)ayhVCt~%wa
z1mK@x*_&Y)Zr-R8vL5y4K^C&n-YP?c~9PQbQiAQGFw0K^SPh``{anBb*9
ztu2?1O$OIYUi`1A3+Xq?An1TGugHcSRuZHVdMCjyTNF^$GePv384$x=c>8I0uX@JV4TUnn6&T7B)f;wXys6S`#
zZa$v4_YGzUNvt}nD0Dc(+9@{euPE9{MD5BStHH9+F6&jN$&Z7Sy?VrF1l9aG8K*r~uJ
z`N1ve{2&Fn%mh7`#Sr+#B=Ei`9r0+LkbvEz-gf7k7}LRi0*C1+$uW@@%wku$4T%!U
zX~04guEh#m)lR3gx6SRtIUU&M;WfIs0yJ(<8ETTjEftD1e?#1F5xhn2Ka6@4Vdd19
zd-`VSIVpMaB;Wcz@}AprHE7lp5w2L$r=`9>?|9Nb*MQ}7SX#H}R^3`h+9&ba8n5lE
z$qt+5F%0eIV$NC=oV5!R!7KSQ(jDf6fplS=<8!#_hBm7{gm%?zYLkqOnB<-R
z8}8nbEJ57x?@ph|$K$8FLsYdz+6&nD>q~~8?)qqp
zp1;66_tKnV2A8S*4wK=n-z49Zc?QA!ZYEPBowG%KJ`}Diz18BBTMM7Gm{dA1f~Ts+
z+t_q6-gvCu=mGeD2Y_w>r1_aof#Ku-VWhO@rAph(*mPi+QXWnN7}~x5o#&eA-PLid
zZoZD$cW07_JfZpN^aqcJ7vDD?*(7apw`5zB^55h~wWXvEeKP-1E<#y{HVp;{j)$O>
z+83U-|7})@X_VI0!Wup0j^6Ij`+><}FbH(DVy?!Ls8CurvzReV&siPDEAcW#4wI1M
z$D8%gS9(MF2?q}ho`+M2cNI11f{7nTFa3wF^=hhh%jC<{V7;GRUaJae1FMMiesTyJ
z)RNt?I(t^mj#(tOS;%B)zk2EEkoU0&1;z<9uDyeB%Sl30TeGy;tBLv4$D8xC3$4dx
zqaYa|54X3DGhQIQv(q-dqI%6xW$AVctGoX&s?+F1o_-dL8`(U@OkGHX@6hj@
zuXRZ*HspDmmhorb!VcOpR;YkAlm2A7xGl5P4VPjnnB$B*nJzTM&hLgkHya%bAKv-%
zRo&RPXwR`%X^*%H-8#4BFAAA##9b6lv9>05DI`v=V~t(vn!K+zqm4P85%KfHC_?5%
zdZME}QWj4_1DpGICzatfT30bA;q`-WzZzU1c6nXbe*u}D#eII(+?TQT|Y8ON@RVnRx&
z?-_p)WpLkA1C&%RIkmrb8%nJzNv_H=R%-mFxulQZ^z6gU-7oeKeW%F~OS_$cA$ekI
z<`MEugNIq|BqD8C;OjU$8~JzrLG``oL`8YydAW}tw`F%^;$9@Q#v4xjm2YM+A3q2I
zI4{Tf5S02Js@h1uE{?2+Hw?NP1#lOP6hQN@h+R&TnzYH)
zF-$Yf4R>>dm_j{pa#XZ<*CQLkA{UPCFxBmG
zP*KqH{RT~ooZ$oN?Q6~2CD`KbhYD_}L;BNsERpK$%IfAHiJi+L#14zpX3`br4`OyV
z`Hr>WJ?~+@`*NU|o>}LXShA1A`{$Gv#CbgnRavM4u<`u*Ob=mK)Y3YVs@|S@IaIHz
za(6Q1!JGK>#V>5LM|n?@c>{NORpbi?r0%-|HT~6#w6$(D8%qz}F847-y(HnkTw$
z`toVk4kTk4sO<8f06|c3M4w!^11DZhe~#-1H5Fa2ymGSD)QhSVYe(jYlsTt3)3w!6
zi!vhprj58IPToXfF|*%cY0jp4jKC?CHEJrg`ARpIM7q;ZRbQ~rB%b?)GQ#xX0Th&^*-XhTO|$EQ15#cq8+uH^myFw
zjr>5nS-r37we3s5vA1A-&E+`1_G(w)h;3=CS+3`uG)tj$p#q9;|
zye>1HU0YUm+i6`IrJiclE^)50uxHzimOL;=1L0FmWpwifuBW^=5+s*Dy80sv&XLD&
zes`(sq{|AecFCKiY*k;=w`S2JyBIxLa7;jnd1T8Myl)>l&9s@sKilI&sbh)vp07GWN*dtB%J#U*pWci
zLVUCOYfx0fW!Q)BY*EZYw)t5rH`*47#tv5WMq|u!2jhlC(tKmQ9gtD^se8TSykzd+
z7#+6GRv)38x$1dCv2L`flCD`OCT0sN}(#m4gY3#F)p2snZoAu;#%_=91ngs*p*%p0cP3whoT)w~1PLgjy
z)VqnqPoG>)=rBN0SCRy5>rhM7PZgZ5_;+jTH^y7l2i|s2r5j%9wT#;b_kSa!OtYTl
z8n3-~UQ&SmdL^w-NYD^qAl|G;uM3tDry+I_D^!gUOhg&!8qTcbKG^ZLED~TO6bnre
z^C}JXqq5t-XYM5_bZ%6q*(x5bDEA3QE(25t90ELoG*6v&i5y7D4cD+pVskxwT?iWI
z7tzlVZJaGSw>{6iI4kJyb$Q#a`zML`ZdH7`5%h2Z4|834Lv?XE;mWzZe*f~p2h4j5
zR^vX~bUqeI(@xYLux3$bXqNxKHWF}h7W<9cX1$f&`f=oPa3)%!jhqFk=LzzwlQgftQBK+MlW)-sJNvb>p!!>uG4j=9Z?vfs|CH&Qt!NeV8LDz@GMgzI=JmAy(jcJ2v50=k@edCO>}^`V%hn
z7wigomrf+MDNCZnwWqJ+)4;Hfv9A6u0Pv6op=Mp6
z%Hovo)L@kNGrLf$dbsc?tNExyoVkh)Dh((>OY=QQsgryY2zY5z4|qsFT)!q)3HvF|
z*vGf^8&OJbF)-2wu##F)5*FXYrG0=dKNt`5b-D#yE8nb7G?|_xXSkCrkyg@K`&Oj)
zBI~bI2BRw12b$eY0CuL|+acZVzc~
z0pPYvC^~)yP_j7PfPuM1`Zpx>+Y{orE4CU)YThON>(mZ=xIyov<_(Lf-mm_InnQq0
z#d-roVnFy*YCt5`;QAw_U(pCL_X-WxrNRn85Mr@+c0Wp>%%$A2Aq;(TdOeo~T@e64
zOBT}nm-V}3k$!c3Oi9Fo`(!faGu|$rX@mT4)+17k2Y$fU_AsS|sBXmw61*6Eea^qD
zL869roHO>DmS(t*gJKF-<|x@E%{*5pvSWWhj~