RAFT-Model-Comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Enable asyncio in Jupyter Notebook\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from datasets import load_dataset\n",
    "from langchain.evaluation import CotQAEvalChain, load_evaluator\n",
    "from langchain.prompts import ChatPromptTemplate\n",
    "from langchain.smith import RunEvalConfig, arun_on_dataset\n",
    "from langchain_anthropic.chat_models import ChatAnthropic\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_openai.chat_models import ChatOpenAI\n",
    "from langsmith import Client\n",
    "\n",
    "from src.settings import settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1399)\n",
    "langsmith_client = Client()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Load models to evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define generation arguments for all the models\n",
    "generation_kwargs = {\n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 500\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# VLLM endpoint for the RAFT fine tuned model\n",
    "endpoint = \"https://jjovalle99--raft-starling7b-ft-serve-model.modal.run/v1\"\n",
    "raft_starling_7b = ChatOpenAI(\n",
    "    model=\"jjovalle99/starling-7b-raft-ft\",\n",
    "    openai_api_key=\"EMPTY\",\n",
    "    openai_api_base=endpoint,\n",
    "    **generation_kwargs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comparison models\n",
    "openai_llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", api_key=settings.env.OPENAI_API_KEY, **generation_kwargs)\n",
    "anthropic_llm = ChatAnthropic(model=\"claude-3-haiku-20240307\", anthropic_api_key=settings.env.ANTHROPIC_API_KEY, **generation_kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading created dataset\n",
    "dataset = load_dataset(\"jjovalle99/raft-dataset-aws-wellarchitected\", split=\"train\")\n",
    "# Sampling 30 examples\n",
    "sampled_dataset = dataset.shuffle(seed=1399).select(range(30))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare context\n",
    "sampled_dataset = sampled_dataset.map(lambda x: {\"context\": \"\\n\".join([f\"<DOCUMENT>{str(doc)}</DOCUMENT>\" for doc in x[\"context\"][\"sentences\"][0]])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dataset in LangSmith\n",
    "dataset_name = \"RAFT\"\n",
    "dataset = langsmith_client.create_dataset(\n",
    "    dataset_name=dataset_name,\n",
    "    description=\"This dataset is used for evaluating the RAFT model.\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Populate dataset\n",
    "instruction = \"Carefully read and analyze the provided documents to answer the question that follows. \" \\\n",
    "\"Provide a detailed, step-by-step explanation of your reasoning, demonstrating how you arrived at your\" \\\n",
    "\" conclusion based on the information given in the documents.\\n\\n\"\n",
    "\n",
    "dataset_inputs = [\n",
    "    {\n",
    "        \"context\": f\"{instruction}{example['context']}\",\n",
    "        \"question\": example[\"question\"],\n",
    "        \"answer\": example[\"cot_answer\"]\n",
    "    } \n",
    "    for example in sampled_dataset\n",
    "]\n",
    "dataset_outputs = [{\"answer\": example[\"cot_answer\"]} for example in sampled_dataset]\n",
    "\n",
    "langsmith_client.create_examples(\n",
    "    inputs=dataset_inputs,\n",
    "    outputs=dataset_outputs,\n",
    "    dataset_id=dataset.id,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![img](assets/dataset-langsmith2.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Evaluation With Context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluator llm\n",
    "eval_llm = ChatOpenAI(model=\"gpt-4-turbo-preview\", api_key=settings.env.OPENAI_API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define chain\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    messages=[(\"user\", f\"{instruction}{{context}}\\n{{question}}\")]\n",
    ")\n",
    "\n",
    "output_parser = StrOutputParser()\n",
    "\n",
    "def create_chain_by_model(model):\n",
    "    return (\n",
    "        prompt \n",
    "        | model \n",
    "        | output_parser\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 How are we evaluating?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "_example = sampled_dataset[0]\n",
    "_output = create_chain_by_model(model=raft_starling_7b).invoke({\n",
    "    \"context\": _example[\"context\"],\n",
    "    \"question\": _example[\"question\"]\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.1.1 How is Coherence evaluated?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'langchain.evaluation.criteria.eval_chain.CriteriaEvalChain'>\n",
      "You are assessing a submitted answer on a given task or input based on a set of criteria. Here is the data:\n",
      "[BEGIN DATA]\n",
      "***\n",
      "[Input]: This is a mock input\n",
      "***\n",
      "[Submission]: This is a mock output\n",
      "***\n",
      "[Criteria]: coherence: Is the submission coherent, well-structured, and organized?\n",
      "***\n",
      "[END DATA]\n",
      "Does the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character \"Y\" or \"N\" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line.\n"
     ]
    }
   ],
   "source": [
    "evaluator = load_evaluator(\"criteria\", criteria=\"coherence\")\n",
    "print(type(evaluator))\n",
    "print(evaluator.prompt.invoke({\"input\": \"This is a mock input\", \"output\": \"This is a mock output\"}).text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input': \"<DOCUMENT>They allow communication between instances in your VPC and services without imposing availability risks or bandwidth constraints on your network traffic. ## Common anti-patterns:\\n\\nFoundations - 415\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Having only one connectivity provider between your on-site network and AWS. - Consuming the connectivity capabilities of your AWS Direct Connect connection, but only having one connection. - Having only one path for your VPN connectivity. Benefits of establishing this best practice: By implementing redundant connectivity between your cloud environment and your corporate or on-premises environment, you can ensure that the dependent services between the two environments can communicate reliably. Level of risk exposed if this best practice is not established: High\\n\\n### Implementation guidance\\n\\n- Ensure that you have highly available connectivity between AWS and on-premises environment. Use multiple AWS Direct Connect connections or VPN tunnels between separately deployed private networks. Use multiple Direct Connect locations for high availability. If using multiple AWS Regions, ensure redundancy in at least two of them. You might want to evaluate AWS Marketplace appliances that terminate VPNs. If you use AWS Marketplace appliances, deploy redundant instances for high availability in different Availability Zones. - Ensure that you have a redundant connection to your on-premises environment. You may need redundant connections to multiple AWS Regions to achieve your availability needs. - AWS Direct Connect Resiliency Recommendations\\n- Using Redundant Site-to-Site VPN Connections to Provide Failover\\n- Use service API operations to identify correct use of Direct Connect circuits. - DescribeConnections\\n- DescribeConnectionsOnInterconnect\\n- DescribeDirectConnectGatewayAssociations\\n- DescribeDirectConnectGatewayAttachments\\n- DescribeDirectConnectGateways\\n- DescribeHostedConnections\\n- DescribeInterconnects\\n- If only one Direct Connect connection exists or you have none, set up redundant VPN tunnels to your virtual private gateways. ### Foundations\\n\\nWhat is AWS Site-to-Site VPN? 416\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n### Capture your current connectivity (for example, Direct Connect, virtual private gateways, AWS Marketplace appliances). - Use service API operations to query configuration of Direct Connect connections. - DescribeConnections\\n- DescribeConnectionsOnInterconnect\\n- DescribeDirectConnectGatewayAssociations\\n- DescribeDirectConnectGatewayAttachments\\n- DescribeDirectConnectGateways\\n- DescribeHostedConnections\\n- DescribeInterconnects\\n- Use service API operations to collect virtual private gateways where route tables use them. - DescribeVpnGateways\\n- DescribeRouteTables\\n- Use service API operations to collect AWS Marketplace applications where route tables use them. - DescribeRouteTables\\n\\n## Resources\\n\\nRelated documents:\\n\\n- APN Partner: partners that can help plan your networking\\n- AWS Direct Connect Resiliency Recommendations\\n- AWS Marketplace for Network Infrastructure\\n- Amazon Virtual Private Cloud Connectivity Options Whitepaper\\n- Multiple data center HA network connectivity\\n- Using Redundant Site-to-Site VPN Connections to Provide Failover\\n- Using the Direct Connect Resiliency Toolkit to get started\\n- VPC Endpoints and VPC Endpoint Services (AWS PrivateLink)\\n- What Is Amazon VPC? - What Is a Transit Gateway?</DOCUMENT>\\n<DOCUMENT>This also places addition operational burden on you as you restore the missing data. Benefits of establishing this best practice: Having a predefined processes to determine the components, conditions, actions, and events that contributed to an incident helps you to identify opportunities for improvement. Level of risk exposed if this best practice is not established: High\\n\\n### Implementation guidance\\n\\n- Use a process to determine contributing factors: Review all customer impacting incidents. Have a process to identify and document the contributing factors of an incident so that you can develop mitigations to limit or prevent recurrence and you can develop procedures for prompt and effective responses. Communicate root cause as appropriate, tailored to target audiences. #### OPS11-BP03 Implement feedback loops\\n\\nFeedback loops provide actionable insights that drive decision making. Build feedback loops into your procedures and workloads. This helps you identify issues and areas that need improvement. They also validate investments made in improvements. These feedback loops are the foundation for continuously improving your workload. Feedback loops fall into two categories: immediate feedback and retrospective analysis. Immediate feedback is gathered through review of the performance and outcomes from operations activities. This feedback comes from team members, customers, or the automated output of the activity. Immediate feedback is received from things like A/B testing and shipping new features, and it is essential to failing fast. Retrospective analysis is performed regularly to capture feedback from the review of operational outcomes and metrics over time. These retrospectives happen at the end of a sprint, on a cadence, or after major releases or events. This type of feedback loop validates investments in operations or your workload. It helps you measure success and validates your strategy. ## Evolve\\n\\n208\\n---\\n## AWS Well-Architected Framework\\n\\nDesired outcome: You use immediate feedback and retrospective analysis to drive improvements. There is a mechanism to capture user and team member feedback.</DOCUMENT>\\n<DOCUMENT>You can also map specific accounts and tags to multiple projects. ## Implementation steps\\n\\n- Define a tagging schema: Gather all stakeholders from across your business to define a schema. This typically includes people in technical, financial, and management roles. Define a list of tags that all resources must have, as well as a list of tags that resources should have. Verify that the tag names and values are consistent across your organization. - Tag resources: Using your defined cost attribution categories, place tags on all resources in your workloads according to the categories. Use tools such as the CLI, Tag Editor, or AWS Systems Manager to increase efficiency. - Implement AWS Cost Categories: You can create Cost Categories without implementing tagging. Cost categories use the existing cost and usage dimensions. Create category rules from your schema and implement them into cost categories. - Automate tagging: To verify that you maintain high levels of tagging across all resources, automate tagging so that resources are automatically tagged when they are created. Use services such as AWS CloudFormation to verify that resources are tagged when created. You can also create a custom solution to tag automatically using Lambda functions or use a microservice that\\n\\n## Expenditure and usage awareness\\n\\n739\\n---\\n## AWS Well-Architected Framework\\n\\nThe framework scans the workload periodically and removes any resources that are not tagged, which is ideal for test and development environments. - Monitor and report on tagging: To verify that you maintain high levels of tagging across your organization, report and monitor the tags across your workloads. You can use AWS Cost Explorer to view the cost of tagged and untagged resources, or use services such as Tag Editor. Regularly review the number of untagged resources and take action to add tags until you reach the desired level of tagging. ### Resources\\n\\nRelated documents:\\n\\n- Tagging Best Practices\\n- AWS CloudFormation Resource Tag\\n- AWS Cost Categories\\n- Tagging AWS resources\\n- Analyzing your costs with AWS Budgets\\n- Analyzing your costs with Cost Explorer\\n- Managing AWS Cost and Usage Reports\\n\\nRelated videos:\\n\\n- How can I tag my AWS resources to divide up my bill by cost center or project\\n- Tagging AWS Resources\\n\\nRelated examples:\\n\\n- Automatically tag new AWS resources based on identity or role\\n\\n### COST03-BP03 Identify cost attribution categories\\n\\nIdentify organization categories such as business units, departments or projects that could be used to allocate cost within your organization to the internal consuming entities. Use those categories to enforce spend accountability, create cost awareness and drive effective consumption behaviors. Level of risk exposed if this best practice is not established: High\\n\\nExpenditure and usage awareness: 740\\n---\\n## AWS Well-Architected Framework\\n\\nImplementation guidance\\n\\nThe process of categorizing costs is crucial in budgeting, accounting, financial reporting, decision making, benchmarking, and project management. By classifying and categorizing expenses, teams can gain a better understanding of the types of costs they incur throughout their cloud journey helping teams make informed decisions and manage budgets effectively. Cloud spend accountability establishes a strong incentive for disciplined demand and cost management. The result is significantly greater cloud cost savings for organizations that allocate most of their cloud spend to consuming business units or teams. Moreover, allocating cloud spend helps organizations adopt more best practices of centralized cloud governance. Work with your finance team and other relevant stakeholders to understand the requirements of how costs must be allocated within your organization during your regular cadence calls. Workload costs must be allocated throughout the entire lifecycle, including development, testing, production, and decommissioning. Understand how the costs incurred for learning, staff development, and idea creation are attributed in the organization. This can be helpful to correctly allocate accounts used for this purpose to training and development budgets instead of generic IT cost budgets. After defining your cost attribution categories with stakeholders in your organization, use AWS Cost Categories to group your cost and usage information into meaningful categories in the AWS Cloud, such as cost for a specific project, or AWS accounts for departments or business units. You can create custom categories and map your cost and usage information into these categories based on rules you define using various dimensions such as account, tag, service, or charge type. Once cost categories are set up, you can view your cost and usage information by these categories, which allows your organization to make better strategic and purchasing decisions. These categories are visible in AWS Cost Explorer, AWS Budgets, and AWS Cost and Usage Report as well. For example, create cost categories for your business units (DevOps team), and under each category create multiple rules (rules for each sub category) with multiple dimensions (AWS accounts, cost allocation tags, services or charge type) based on your defined groupings. With cost categories, you can organize your costs using a rule-based engine. The rules that you configure organize your costs into categories. Within these rules, you can filter with using multiple dimensions for each category such as specific AWS accounts, AWS services, or charge types. You can then use these categories across multiple products in the AWS Billing and Cost Management and Cost Management console. This includes AWS Cost Explorer, AWS Budgets, AWS Cost and Usage Report, and AWS Cost Anomaly Detection. Expenditure and usage awareness\\n\\n741\\n---\\n## AWS Well-Architected Framework\\n\\nAs an example, the following diagram displays how to group your costs and usage information in your organization by having multiple teams (cost category), multiple environments (rules), and each environment having multiple resources or assets (dimensions). |Team| | |\\n|---|---|---|\\n|production|testing|staging|\\n|accountID 1|accountID 2|accountID 3|\\n|Cost allocation tags|Cost allocation tags|Cost allocation tags|\\n|name: project1|name: project1|name: project1|\\n|name: project2|name: project2|name: project2|\\n|name: project3|name: project3|name: project3|\\n\\nCost and usage organization chart\\n\\nYou can create groupings of costs using cost categories as well. After you create the cost categories (allowing up to 24 hours after creating a cost category for your usage records to be updated with values), they appear in AWS Cost Explorer, AWS Budgets, AWS Cost and Usage Report, and AWS Cost Anomaly Detection. In AWS Cost Explorer and AWS Budgets, a cost category appears as an additional billing dimension. You can use this to filter for the specific cost category value, or group by the cost category. ### Implementation steps\\n\\n- Define your organization categories: Meet with internal stakeholders and business units to define categories that reflect your organization's structure and requirements. These categories should directly map to the structure of existing financial categories, such as business unit, budget, cost center, or department. Look at the outcomes the cloud delivers for your business such as training or education, as these are also organization categories. - Define your functional categories: Meet with internal stakeholders and business units to define categories that reflect the functions that you have within your business. This may be the Expenditure and usage awareness\\n---\\n## AWS Well-Architected Framework\\n\\nworkload or application names, and the type of environment, such as production, testing, or development. - Define AWS Cost Categories: Create cost categories to organize your cost and usage information with using AWS Cost Categories and map your AWS cost and usage into meaningful categories. Multiple categories can be assigned to a resource, and a resource can be in multiple different categories, so define as many categories as needed so that you can manage your costs within the categorized structure using AWS Cost Categories. ### Resources\\n\\nRelated documents:\\n\\n- Tagging AWS resources\\n- Using Cost Allocation Tags\\n- Analyzing your costs with AWS Budgets\\n- Analyzing your costs with Cost Explorer\\n- Managing AWS Cost and Usage Reports\\n- AWS Cost Categories\\n- Managing your costs with AWS Cost Categories\\n- Creating cost categories\\n- Tagging cost categories\\n- Splitting charges within cost categories\\n- AWS Cost Categories Features\\n\\nRelated examples:\\n\\n- Organize your cost and usage data with AWS Cost Categories\\n- Managing your costs with AWS Cost Categories\\n- Well-Architected Labs: Cost and Usage Visualization\\n- Well-Architected Labs: Cost Categories\\n\\nExpenditure and usage awareness 743\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\nCOST03-BP04 Establish organization metrics\\n\\nEstablish the organization metrics that are required for this workload. Example metrics of a workload are customer reports produced, or web pages served to customers. Level of risk exposed if this best practice is not established: High\\n\\nImplementation guidance\\n\\nUnderstand how your workload’s output is measured against business success. Each workload typically has a small set of major outputs that indicate performance. If you have a complex workload with many components, then you can prioritize the list, or define and track metrics for each component. Work with your teams to understand which metrics to use. This unit will be used to understand the efficiency of the workload, or the cost for each business output. Implementation steps\\n\\n- Define workload outcomes: Meet with the stakeholders in the business and define the outcomes for the workload. These are a primary measure of customer usage and must be business metrics and not technical metrics. There should be a small number of high-level metrics (less than five) per workload. If the workload produces multiple outcomes for different use cases, then group them into a single metric. - Define workload component outcomes: Optionally, if you have a large and complex workload, or can easily break your workload into components (such as microservices) with well-defined inputs and outputs, define metrics for each component. The effort should reflect the value and cost of the component.</DOCUMENT>\\n<DOCUMENT>For more information about getting access to AWS Support features, see Getting started wip AWS Support. AWS Customer Incident Response Team (CIRT)\\n\\nIncident response\\n\\n349\\n---\\n## AWS Well-Architected Framework\\n\\nThe AWS Customer Incident Response Team (CIRT) is a specialized 24/7 global AWS team that provides support to customers during active security events on the customer side of the AWS Shared Responsibility Model. When the AWS CIRT supports you, they provide assistance with triage and recovery for an active security event on AWS. They can assist in root cause analysis through the use of AWS service logs and provide you with recommendations for recovery. They can also provide security recommendations and best practices to help you avoid security events in the future. AWS customers can engage the AWS CIRT through an AWS Support case. ## DDoS response support\\n\\nAWS offers AWS Shield, which provides a managed distributed denial of service (DDoS) protection service that safeguards web applications running on AWS. Shield provides always-on detection and automatic inline mitigations that can minimize application downtime and latency, so there is no need to engage AWS Support to benefit from DDoS protection. There are two tiers of Shield: AWS Shield Standard and AWS Shield Advanced. To learn about the differences between these two tiers, see Shield features documentation. ## AWS Managed Services (AMS)\\n\\nAWS Managed Services (AMS) provides ongoing management of your AWS infrastructure so you can focus on your applications. By implementing best practices to maintain your infrastructure, AMS helps reduce your operational overhead and risk. AMS automates common activities such as change requests, monitoring, patch management, security, and backup services, and provides full-lifecycle services to provision, run, and support your infrastructure. AMS takes responsibility for deploying a suite of security detective controls and provides a 24/7 first line of response to alerts. When an alert is initiated, AMS follows a standard set of automated and manual playbooks to verify a consistent response. These playbooks are shared with AMS customers during onboarding so that they can develop and coordinate a response with AMS. ## Develop the incident response plan\\n\\nThe incident response plan is designed to be the foundation for your incident response program and strategy. The incident response plan should be in a formal document. An incident response plan typically includes these sections:\\n\\n- An incident response team overview: Outlines the goals and functions of the incident response team. ## Incident response\\n\\n350\\n---\\n## AWS Well-Architected Framework\\n\\n|Roles and responsibilities|Lists the incident response stakeholders and details their roles when an incident occurs.|\\n|---|---|\\n|A communication plan|Details contact information and how you communicate during an incident.|\\n|Backup communication methods|It’s a best practice to have out-of-band communication as a backup for incident communication. An example of an application that provides a secure out-of-band communications channel is AWS Wickr.|\\n|Phases of incident response and actions to take|Enumerates the phases of incident response (for example, detect, analyze, eradicate, contain, and recover), including high-level actions to take within those phases.|\\n|Incident severity and prioritization definitions|Details how to classify the severity of an incident, how to prioritize the incident, and then how the severity definitions affect escalation procedures.|\\n\\nWhile these sections are common throughout companies of different sizes and industries, each organization’s incident response plan is unique. You need to build an incident response plan that works best for your organization. ## Resources\\n\\nRelated best practices:\\n\\n- SEC04 (How do you detect and investigate security events?)\\n\\nRelated documents:\\n\\n- AWS Security Incident Response Guide\\n- NIST: Computer Security Incident Handling Guide\\n\\nSEC10-BP03 Prepare forensic capabilities\\n\\nAhead of a security incident, consider developing forensics capabilities to support security event investigations. Level of risk exposed if this best practice is not established: Medium\\n\\nIncident response 351\\n---\\n## AWS Well-Architected Framework\\n\\nConcepts from traditional on-premises forensics apply to AWS. For key information to start building forensics capabilities in the AWS Cloud, see Forensic investigation environment strategies in the AWS Cloud. Once you have your environment and AWS account structure set up for forensics, define the technologies required to effectively perform forensically sound methodologies across the four phases:\\n\\n- Collection: Collect relevant AWS logs, such as AWS CloudTrail, AWS Config, VPC Flow Logs, and host-level logs. Collect snapshots, backups, and memory dumps of impacted AWS resources where available. - Examination: Examine the data collected by extracting and assessing the relevant information. - Analysis: Analyze the data collected in order to understand the incident and draw conclusions from it.</DOCUMENT>\",\n",
       " 'output': ' ##Reason: The document states ##begin_quote## Level of risk exposed if this best practice is not established: High ##end_quote## which directly answers the question by indicating the level of risk associated with not following the best practice of establishing predefined processes for incident analysis. Therefore, the level of risk if the best practice is not established is high. ##Answer: High',\n",
       " 'results': {'reasoning': 'The submission is being evaluated on the criterion of coherence, which involves assessing whether the submission is well-structured, organized, and makes logical sense.\\n\\n1. Coherence: The submission begins by stating the reason for the answer, which is a quote from the document. The quote is relevant and directly answers the question about the level of risk associated with not following a certain best practice. The submission then provides the answer, which is consistent with the quote provided. The submission is logically structured, with the reason provided first followed by the answer. The submission is also organized, with clear separation between the reason and the answer. The submission makes logical sense, as the answer is directly supported by the quote from the document.\\n\\nBased on this analysis, the submission meets the criterion of coherence. \\n\\nThe answer is: \\n\\nY',\n",
       "  'value': 'Y',\n",
       "  'score': 1}}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.invoke({\"input\": _example[\"context\"], \"output\": _output})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.1.2 How is Helpfulness evaluated?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'langchain.evaluation.criteria.eval_chain.CriteriaEvalChain'>\n",
      "You are assessing a submitted answer on a given task or input based on a set of criteria. Here is the data:\n",
      "[BEGIN DATA]\n",
      "***\n",
      "[Input]: This is a mock input\n",
      "***\n",
      "[Submission]: This is a mock output\n",
      "***\n",
      "[Criteria]: helpfulness: Is the submission helpful, insightful, and appropriate? If so, respond Y. If not, respond N.\n",
      "***\n",
      "[END DATA]\n",
      "Does the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character \"Y\" or \"N\" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line.\n"
     ]
    }
   ],
   "source": [
    "evaluator = load_evaluator(\"criteria\", criteria=\"helpfulness\")\n",
    "print(type(evaluator))\n",
    "print(evaluator.prompt.invoke({\"input\": \"This is a mock input\", \"output\": \"This is a mock output\"}).text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input': \"<DOCUMENT>They allow communication between instances in your VPC and services without imposing availability risks or bandwidth constraints on your network traffic. ## Common anti-patterns:\\n\\nFoundations - 415\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Having only one connectivity provider between your on-site network and AWS. - Consuming the connectivity capabilities of your AWS Direct Connect connection, but only having one connection. - Having only one path for your VPN connectivity. Benefits of establishing this best practice: By implementing redundant connectivity between your cloud environment and your corporate or on-premises environment, you can ensure that the dependent services between the two environments can communicate reliably. Level of risk exposed if this best practice is not established: High\\n\\n### Implementation guidance\\n\\n- Ensure that you have highly available connectivity between AWS and on-premises environment. Use multiple AWS Direct Connect connections or VPN tunnels between separately deployed private networks. Use multiple Direct Connect locations for high availability. If using multiple AWS Regions, ensure redundancy in at least two of them. You might want to evaluate AWS Marketplace appliances that terminate VPNs. If you use AWS Marketplace appliances, deploy redundant instances for high availability in different Availability Zones. - Ensure that you have a redundant connection to your on-premises environment. You may need redundant connections to multiple AWS Regions to achieve your availability needs. - AWS Direct Connect Resiliency Recommendations\\n- Using Redundant Site-to-Site VPN Connections to Provide Failover\\n- Use service API operations to identify correct use of Direct Connect circuits. - DescribeConnections\\n- DescribeConnectionsOnInterconnect\\n- DescribeDirectConnectGatewayAssociations\\n- DescribeDirectConnectGatewayAttachments\\n- DescribeDirectConnectGateways\\n- DescribeHostedConnections\\n- DescribeInterconnects\\n- If only one Direct Connect connection exists or you have none, set up redundant VPN tunnels to your virtual private gateways. ### Foundations\\n\\nWhat is AWS Site-to-Site VPN? 416\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n### Capture your current connectivity (for example, Direct Connect, virtual private gateways, AWS Marketplace appliances). - Use service API operations to query configuration of Direct Connect connections. - DescribeConnections\\n- DescribeConnectionsOnInterconnect\\n- DescribeDirectConnectGatewayAssociations\\n- DescribeDirectConnectGatewayAttachments\\n- DescribeDirectConnectGateways\\n- DescribeHostedConnections\\n- DescribeInterconnects\\n- Use service API operations to collect virtual private gateways where route tables use them. - DescribeVpnGateways\\n- DescribeRouteTables\\n- Use service API operations to collect AWS Marketplace applications where route tables use them. - DescribeRouteTables\\n\\n## Resources\\n\\nRelated documents:\\n\\n- APN Partner: partners that can help plan your networking\\n- AWS Direct Connect Resiliency Recommendations\\n- AWS Marketplace for Network Infrastructure\\n- Amazon Virtual Private Cloud Connectivity Options Whitepaper\\n- Multiple data center HA network connectivity\\n- Using Redundant Site-to-Site VPN Connections to Provide Failover\\n- Using the Direct Connect Resiliency Toolkit to get started\\n- VPC Endpoints and VPC Endpoint Services (AWS PrivateLink)\\n- What Is Amazon VPC? - What Is a Transit Gateway?</DOCUMENT>\\n<DOCUMENT>This also places addition operational burden on you as you restore the missing data. Benefits of establishing this best practice: Having a predefined processes to determine the components, conditions, actions, and events that contributed to an incident helps you to identify opportunities for improvement. Level of risk exposed if this best practice is not established: High\\n\\n### Implementation guidance\\n\\n- Use a process to determine contributing factors: Review all customer impacting incidents. Have a process to identify and document the contributing factors of an incident so that you can develop mitigations to limit or prevent recurrence and you can develop procedures for prompt and effective responses. Communicate root cause as appropriate, tailored to target audiences. #### OPS11-BP03 Implement feedback loops\\n\\nFeedback loops provide actionable insights that drive decision making. Build feedback loops into your procedures and workloads. This helps you identify issues and areas that need improvement. They also validate investments made in improvements. These feedback loops are the foundation for continuously improving your workload. Feedback loops fall into two categories: immediate feedback and retrospective analysis. Immediate feedback is gathered through review of the performance and outcomes from operations activities. This feedback comes from team members, customers, or the automated output of the activity. Immediate feedback is received from things like A/B testing and shipping new features, and it is essential to failing fast. Retrospective analysis is performed regularly to capture feedback from the review of operational outcomes and metrics over time. These retrospectives happen at the end of a sprint, on a cadence, or after major releases or events. This type of feedback loop validates investments in operations or your workload. It helps you measure success and validates your strategy. ## Evolve\\n\\n208\\n---\\n## AWS Well-Architected Framework\\n\\nDesired outcome: You use immediate feedback and retrospective analysis to drive improvements. There is a mechanism to capture user and team member feedback.</DOCUMENT>\\n<DOCUMENT>You can also map specific accounts and tags to multiple projects. ## Implementation steps\\n\\n- Define a tagging schema: Gather all stakeholders from across your business to define a schema. This typically includes people in technical, financial, and management roles. Define a list of tags that all resources must have, as well as a list of tags that resources should have. Verify that the tag names and values are consistent across your organization. - Tag resources: Using your defined cost attribution categories, place tags on all resources in your workloads according to the categories. Use tools such as the CLI, Tag Editor, or AWS Systems Manager to increase efficiency. - Implement AWS Cost Categories: You can create Cost Categories without implementing tagging. Cost categories use the existing cost and usage dimensions. Create category rules from your schema and implement them into cost categories. - Automate tagging: To verify that you maintain high levels of tagging across all resources, automate tagging so that resources are automatically tagged when they are created. Use services such as AWS CloudFormation to verify that resources are tagged when created. You can also create a custom solution to tag automatically using Lambda functions or use a microservice that\\n\\n## Expenditure and usage awareness\\n\\n739\\n---\\n## AWS Well-Architected Framework\\n\\nThe framework scans the workload periodically and removes any resources that are not tagged, which is ideal for test and development environments. - Monitor and report on tagging: To verify that you maintain high levels of tagging across your organization, report and monitor the tags across your workloads. You can use AWS Cost Explorer to view the cost of tagged and untagged resources, or use services such as Tag Editor. Regularly review the number of untagged resources and take action to add tags until you reach the desired level of tagging. ### Resources\\n\\nRelated documents:\\n\\n- Tagging Best Practices\\n- AWS CloudFormation Resource Tag\\n- AWS Cost Categories\\n- Tagging AWS resources\\n- Analyzing your costs with AWS Budgets\\n- Analyzing your costs with Cost Explorer\\n- Managing AWS Cost and Usage Reports\\n\\nRelated videos:\\n\\n- How can I tag my AWS resources to divide up my bill by cost center or project\\n- Tagging AWS Resources\\n\\nRelated examples:\\n\\n- Automatically tag new AWS resources based on identity or role\\n\\n### COST03-BP03 Identify cost attribution categories\\n\\nIdentify organization categories such as business units, departments or projects that could be used to allocate cost within your organization to the internal consuming entities. Use those categories to enforce spend accountability, create cost awareness and drive effective consumption behaviors. Level of risk exposed if this best practice is not established: High\\n\\nExpenditure and usage awareness: 740\\n---\\n## AWS Well-Architected Framework\\n\\nImplementation guidance\\n\\nThe process of categorizing costs is crucial in budgeting, accounting, financial reporting, decision making, benchmarking, and project management. By classifying and categorizing expenses, teams can gain a better understanding of the types of costs they incur throughout their cloud journey helping teams make informed decisions and manage budgets effectively. Cloud spend accountability establishes a strong incentive for disciplined demand and cost management. The result is significantly greater cloud cost savings for organizations that allocate most of their cloud spend to consuming business units or teams. Moreover, allocating cloud spend helps organizations adopt more best practices of centralized cloud governance. Work with your finance team and other relevant stakeholders to understand the requirements of how costs must be allocated within your organization during your regular cadence calls. Workload costs must be allocated throughout the entire lifecycle, including development, testing, production, and decommissioning. Understand how the costs incurred for learning, staff development, and idea creation are attributed in the organization. This can be helpful to correctly allocate accounts used for this purpose to training and development budgets instead of generic IT cost budgets. After defining your cost attribution categories with stakeholders in your organization, use AWS Cost Categories to group your cost and usage information into meaningful categories in the AWS Cloud, such as cost for a specific project, or AWS accounts for departments or business units. You can create custom categories and map your cost and usage information into these categories based on rules you define using various dimensions such as account, tag, service, or charge type. Once cost categories are set up, you can view your cost and usage information by these categories, which allows your organization to make better strategic and purchasing decisions. These categories are visible in AWS Cost Explorer, AWS Budgets, and AWS Cost and Usage Report as well. For example, create cost categories for your business units (DevOps team), and under each category create multiple rules (rules for each sub category) with multiple dimensions (AWS accounts, cost allocation tags, services or charge type) based on your defined groupings. With cost categories, you can organize your costs using a rule-based engine. The rules that you configure organize your costs into categories. Within these rules, you can filter with using multiple dimensions for each category such as specific AWS accounts, AWS services, or charge types. You can then use these categories across multiple products in the AWS Billing and Cost Management and Cost Management console. This includes AWS Cost Explorer, AWS Budgets, AWS Cost and Usage Report, and AWS Cost Anomaly Detection. Expenditure and usage awareness\\n\\n741\\n---\\n## AWS Well-Architected Framework\\n\\nAs an example, the following diagram displays how to group your costs and usage information in your organization by having multiple teams (cost category), multiple environments (rules), and each environment having multiple resources or assets (dimensions). |Team| | |\\n|---|---|---|\\n|production|testing|staging|\\n|accountID 1|accountID 2|accountID 3|\\n|Cost allocation tags|Cost allocation tags|Cost allocation tags|\\n|name: project1|name: project1|name: project1|\\n|name: project2|name: project2|name: project2|\\n|name: project3|name: project3|name: project3|\\n\\nCost and usage organization chart\\n\\nYou can create groupings of costs using cost categories as well. After you create the cost categories (allowing up to 24 hours after creating a cost category for your usage records to be updated with values), they appear in AWS Cost Explorer, AWS Budgets, AWS Cost and Usage Report, and AWS Cost Anomaly Detection. In AWS Cost Explorer and AWS Budgets, a cost category appears as an additional billing dimension. You can use this to filter for the specific cost category value, or group by the cost category. ### Implementation steps\\n\\n- Define your organization categories: Meet with internal stakeholders and business units to define categories that reflect your organization's structure and requirements. These categories should directly map to the structure of existing financial categories, such as business unit, budget, cost center, or department. Look at the outcomes the cloud delivers for your business such as training or education, as these are also organization categories. - Define your functional categories: Meet with internal stakeholders and business units to define categories that reflect the functions that you have within your business. This may be the Expenditure and usage awareness\\n---\\n## AWS Well-Architected Framework\\n\\nworkload or application names, and the type of environment, such as production, testing, or development. - Define AWS Cost Categories: Create cost categories to organize your cost and usage information with using AWS Cost Categories and map your AWS cost and usage into meaningful categories. Multiple categories can be assigned to a resource, and a resource can be in multiple different categories, so define as many categories as needed so that you can manage your costs within the categorized structure using AWS Cost Categories. ### Resources\\n\\nRelated documents:\\n\\n- Tagging AWS resources\\n- Using Cost Allocation Tags\\n- Analyzing your costs with AWS Budgets\\n- Analyzing your costs with Cost Explorer\\n- Managing AWS Cost and Usage Reports\\n- AWS Cost Categories\\n- Managing your costs with AWS Cost Categories\\n- Creating cost categories\\n- Tagging cost categories\\n- Splitting charges within cost categories\\n- AWS Cost Categories Features\\n\\nRelated examples:\\n\\n- Organize your cost and usage data with AWS Cost Categories\\n- Managing your costs with AWS Cost Categories\\n- Well-Architected Labs: Cost and Usage Visualization\\n- Well-Architected Labs: Cost Categories\\n\\nExpenditure and usage awareness 743\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\nCOST03-BP04 Establish organization metrics\\n\\nEstablish the organization metrics that are required for this workload. Example metrics of a workload are customer reports produced, or web pages served to customers. Level of risk exposed if this best practice is not established: High\\n\\nImplementation guidance\\n\\nUnderstand how your workload’s output is measured against business success. Each workload typically has a small set of major outputs that indicate performance. If you have a complex workload with many components, then you can prioritize the list, or define and track metrics for each component. Work with your teams to understand which metrics to use. This unit will be used to understand the efficiency of the workload, or the cost for each business output. Implementation steps\\n\\n- Define workload outcomes: Meet with the stakeholders in the business and define the outcomes for the workload. These are a primary measure of customer usage and must be business metrics and not technical metrics. There should be a small number of high-level metrics (less than five) per workload. If the workload produces multiple outcomes for different use cases, then group them into a single metric. - Define workload component outcomes: Optionally, if you have a large and complex workload, or can easily break your workload into components (such as microservices) with well-defined inputs and outputs, define metrics for each component. The effort should reflect the value and cost of the component.</DOCUMENT>\\n<DOCUMENT>For more information about getting access to AWS Support features, see Getting started wip AWS Support. AWS Customer Incident Response Team (CIRT)\\n\\nIncident response\\n\\n349\\n---\\n## AWS Well-Architected Framework\\n\\nThe AWS Customer Incident Response Team (CIRT) is a specialized 24/7 global AWS team that provides support to customers during active security events on the customer side of the AWS Shared Responsibility Model. When the AWS CIRT supports you, they provide assistance with triage and recovery for an active security event on AWS. They can assist in root cause analysis through the use of AWS service logs and provide you with recommendations for recovery. They can also provide security recommendations and best practices to help you avoid security events in the future. AWS customers can engage the AWS CIRT through an AWS Support case. ## DDoS response support\\n\\nAWS offers AWS Shield, which provides a managed distributed denial of service (DDoS) protection service that safeguards web applications running on AWS. Shield provides always-on detection and automatic inline mitigations that can minimize application downtime and latency, so there is no need to engage AWS Support to benefit from DDoS protection. There are two tiers of Shield: AWS Shield Standard and AWS Shield Advanced. To learn about the differences between these two tiers, see Shield features documentation. ## AWS Managed Services (AMS)\\n\\nAWS Managed Services (AMS) provides ongoing management of your AWS infrastructure so you can focus on your applications. By implementing best practices to maintain your infrastructure, AMS helps reduce your operational overhead and risk. AMS automates common activities such as change requests, monitoring, patch management, security, and backup services, and provides full-lifecycle services to provision, run, and support your infrastructure. AMS takes responsibility for deploying a suite of security detective controls and provides a 24/7 first line of response to alerts. When an alert is initiated, AMS follows a standard set of automated and manual playbooks to verify a consistent response. These playbooks are shared with AMS customers during onboarding so that they can develop and coordinate a response with AMS. ## Develop the incident response plan\\n\\nThe incident response plan is designed to be the foundation for your incident response program and strategy. The incident response plan should be in a formal document. An incident response plan typically includes these sections:\\n\\n- An incident response team overview: Outlines the goals and functions of the incident response team. ## Incident response\\n\\n350\\n---\\n## AWS Well-Architected Framework\\n\\n|Roles and responsibilities|Lists the incident response stakeholders and details their roles when an incident occurs.|\\n|---|---|\\n|A communication plan|Details contact information and how you communicate during an incident.|\\n|Backup communication methods|It’s a best practice to have out-of-band communication as a backup for incident communication. An example of an application that provides a secure out-of-band communications channel is AWS Wickr.|\\n|Phases of incident response and actions to take|Enumerates the phases of incident response (for example, detect, analyze, eradicate, contain, and recover), including high-level actions to take within those phases.|\\n|Incident severity and prioritization definitions|Details how to classify the severity of an incident, how to prioritize the incident, and then how the severity definitions affect escalation procedures.|\\n\\nWhile these sections are common throughout companies of different sizes and industries, each organization’s incident response plan is unique. You need to build an incident response plan that works best for your organization. ## Resources\\n\\nRelated best practices:\\n\\n- SEC04 (How do you detect and investigate security events?)\\n\\nRelated documents:\\n\\n- AWS Security Incident Response Guide\\n- NIST: Computer Security Incident Handling Guide\\n\\nSEC10-BP03 Prepare forensic capabilities\\n\\nAhead of a security incident, consider developing forensics capabilities to support security event investigations. Level of risk exposed if this best practice is not established: Medium\\n\\nIncident response 351\\n---\\n## AWS Well-Architected Framework\\n\\nConcepts from traditional on-premises forensics apply to AWS. For key information to start building forensics capabilities in the AWS Cloud, see Forensic investigation environment strategies in the AWS Cloud. Once you have your environment and AWS account structure set up for forensics, define the technologies required to effectively perform forensically sound methodologies across the four phases:\\n\\n- Collection: Collect relevant AWS logs, such as AWS CloudTrail, AWS Config, VPC Flow Logs, and host-level logs. Collect snapshots, backups, and memory dumps of impacted AWS resources where available. - Examination: Examine the data collected by extracting and assessing the relevant information. - Analysis: Analyze the data collected in order to understand the incident and draw conclusions from it.</DOCUMENT>\",\n",
       " 'output': ' ##Reason: The document states ##begin_quote## Level of risk exposed if this best practice is not established: High ##end_quote## which directly answers the question by indicating the level of risk associated with not following the best practice of establishing predefined processes for incident analysis. Therefore, the level of risk if the best practice is not established is high. ##Answer: High',\n",
       " 'results': {'reasoning': 'The criterion for this task is whether the submission is helpful, insightful, and appropriate. \\n\\nLooking at the submission, it provides a direct answer to the question, which is \"High\". The submission also provides a reason for this answer, referencing a quote from the document that directly supports the answer. This shows that the submission is helpful as it provides a clear and direct answer to the question.\\n\\nThe submission is also insightful as it provides a reason for the answer, showing that the person who submitted the answer has understood the document and the context of the question. \\n\\nFinally, the submission is appropriate as it directly answers the question and provides a reason for the answer, which is what the task asked for. \\n\\nTherefore, the submission meets the criterion. \\n\\nY',\n",
       "  'value': 'Y',\n",
       "  'score': 1}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.invoke({\"input\": _example[\"context\"], \"output\": _output})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.1.3 How is Correcteness/QA evaluated?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'langchain.evaluation.qa.eval_chain.CotQAEvalChain'>\n",
      "You are a teacher grading a quiz.\n",
      "You are given a question, the context the question is about, and the student's answer. You are asked to score the student's answer as either CORRECT or INCORRECT, based on the context.\n",
      "Write out in a step by step manner your reasoning to be sure that your conclusion is correct. Avoid simply stating the correct answer at the outset.\n",
      "\n",
      "Example Format:\n",
      "QUESTION: question here\n",
      "CONTEXT: context the question is about here\n",
      "STUDENT ANSWER: student's answer here\n",
      "EXPLANATION: step by step reasoning here\n",
      "GRADE: CORRECT or INCORRECT here\n",
      "\n",
      "Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n",
      "\n",
      "QUESTION: This is a mock question/query\n",
      "CONTEXT: This is a mock context\n",
      "STUDENT ANSWER: This is a mock result\n",
      "EXPLANATION:\n"
     ]
    }
   ],
   "source": [
    "evaluator = CotQAEvalChain.from_llm(llm=eval_llm)\n",
    "print(type(evaluator))\n",
    "print(evaluator.prompt.invoke({\"context\": \"This is a mock context\", \"query\": \"This is a mock question/query\", \"result\": \"This is a mock result\"}).text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To evaluate the student's answer, we need to follow these steps:\n",
      "\n",
      "1. **Identify the Question's Focus**: The question is asking for the level of risk associated with not following the best practice of establishing predefined processes for incident analysis.\n",
      "\n",
      "2. **Locate Relevant Information in the Context**: The context provided in the second <DOCUMENT> mentions, \"Level of risk exposed if this best practice is not established: High.\" This statement directly relates to the question's focus, which is about the risk level when predefined processes for incident analysis are not established.\n",
      "\n",
      "3. **Analyze the Student's Answer**: The student's answer is \"High.\" They have also provided a reasoning that references the exact quote from the context, indicating they have correctly identified and interpreted the relevant information.\n",
      "\n",
      "4. **Compare Student's Answer with the Context**: The student's answer matches the information given in the context. They have accurately cited the portion of the text that provides a direct answer to the question.\n",
      "\n",
      "5. **Conclusion**: Since the student's answer (\"High\") directly matches the information provided in the context, their answer is correct. They have successfully identified that not following the best practice of establishing predefined processes for incident analysis exposes one to a high level of risk.\n",
      "\n",
      "GRADE: CORRECT\n"
     ]
    }
   ],
   "source": [
    "print(evaluator.invoke({\"context\": _example[\"context\"], \"query\": _example[\"question\"], \"result\": _output})[\"text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Defining the evaluation configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define metrics of eval\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        \"cot_qa\",\n",
    "        RunEvalConfig.LabeledCriteria(\n",
    "            criteria=\"helpfulness\",\n",
    "            input_key=\"question\",\n",
    "            prediction_key=\"output\",\n",
    "        ),\n",
    "        RunEvalConfig.LabeledCriteria(\n",
    "            criteria=\"coherence\",\n",
    "            input_key=\"question\",\n",
    "            prediction_key=\"output\",\n",
    "        )\n",
    "    ],\n",
    "    input_key=\"question\",\n",
    "    eval_llm=eval_llm,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 Run evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jjovalle99/starling-7b-raft-ft\n",
      "View the evaluation results for project 'Evaluation - jjovalle99/starling-7b-raft-ft' at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644/compare?selectedSessions=527c1618-5954-4ec1-9119-904284f16933\n",
      "\n",
      "View all tests for Dataset RAFT at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644\n",
      "[-------------------------------------------->     ] 27/30"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Chain failed for example 539f726b-d146-4f19-9e43-09bb2b4bd978 with inputs {'answer': '##Reason: The document provides a specific best practice under the identifier SUS02-BP01, which is to ##begin_quote## Scale workload infrastructure dynamically ##end_quote##. This is further elaborated as using the elasticity of the cloud to scale infrastructure dynamically to match the supply of cloud resources to demand, thereby avoiding overprovisioned capacity in the workload. The recommendation aims to ensure that infrastructure scales with user load, addressing common anti-patterns such as not scaling infrastructure with user load. ##Answer: Scale workload infrastructure dynamically', 'context': \"Carefully read and analyze the provided documents to answer the question that follows. Provide a detailed, step-by-step explanation of your reasoning, demonstrating how you arrived at your conclusion based on the information given in the documents.\\n\\n<DOCUMENT>Provide your team members with devices that support their needs and minimize their sustainability impact. ### Best practices\\n\\n- SUS02-BP01 Scale workload infrastructure dynamically\\n\\n### Alignment to demand\\n\\n817\\n---\\n## AWS Well-Architected Framework\\n\\n|SUS02-BP02|Align SLAs with sustainability goals|\\n|---|---|\\n|SUS02-BP03|Stop the creation and maintenance of unused assets|\\n|SUS02-BP04|Optimize geographic placement of workloads based on their networking requirements|\\n|SUS02-BP05|Optimize team member resources for activities performed|\\n|SUS02-BP06|Implement buffering or throttling to flatten the demand curve|\\n\\n### SUS02-BP01 Scale workload infrastructure dynamically\\n\\nUse elasticity of the cloud and scale your infrastructure dynamically to match supply of cloud resources to demand and avoid overprovisioned capacity in your workload. Common anti-patterns:\\n\\n- You do not scale your infrastructure with user load.</DOCUMENT>\\n<DOCUMENT>Amazon CloudWatch Logs also supports subscriptions that allow data to flow seamlessly to Amazon S3 where you can use or Amazon Athena to query the data. It also supports queries on a large array of formats. See Supported SerDes and Data Formats in the Amazon Athena User Guide for more information. For analysis of huge log file sets, you can run an Amazon EMR cluster to run petabyte-scale analyses. There are a number of tools provided by AWS Partners and third parties that allow for aggregation, processing, storage, and analytics. These tools include New Relic, Splunk, Loggly, Logstash, CloudHealth, and Nagios. However, outside generation of system and application logs is unique to each cloud provider, and often unique to each service. An often-overlooked part of the monitoring process is data management. You need to determine the retention requirements for monitoring data, and then apply lifecycle policies accordingly. Amazon S3 supports lifecycle management at the S3 bucket level. This lifecycle management can be applied differently to different paths in the bucket. Toward the end of the lifecycle, you can transition data to Amazon S3 Glacier for long-term storage, and then expiration after the end of the retention period is reached. The S3 Intelligent-Tiering storage class is designed to optimize costs by automatically moving data to the most cost-effective access tier, without performance impact or operational overhead. Level of risk exposed if this best practice is not established: Medium\\n\\nImplementation guidance\\n\\n- CloudWatch Logs Insights allows you to interactively search and analyze your log data in Amazon CloudWatch Logs. Change management\\n\\n478\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Analyzing Log Data with CloudWatch Logs Insights\\n- Amazon CloudWatch Logs Insights Sample Queries\\n- Use Amazon CloudWatch Logs send logs to Amazon S3 where you can use or Amazon Athena to query the data. - How do I analyze my Amazon S3 server access logs using Athena?</DOCUMENT>\\n<DOCUMENT>Amazon CodeGuru can scan and provide recommendations to remediating common security issues for both Java and Python applications. The OWASP Foundation publishes a list of Source Code Analysis Tools (also known as SAST tools). - Implement a mechanism to scan and patch your existing environment, as well as scanning as part of a CI/CD pipeline build process: Implement a mechanism to scan and patch for issues in your dependencies and operating systems to help protect against new threats. Have that mechanism run on a regular basis. Software vulnerability management is essential to understanding where you need to apply patches or address software issues. Prioritize remediation of potential security issues by embedding vulnerability assessments early into your continuous integration/continuous delivery (CI/CD) pipeline. Your approach can vary based on the AWS services that you are consuming. To check for potential issues in software running in Amazon EC2 instances, add Amazon Inspector to your pipeline to alert you and stop the build process if issues or potential defects are detected. Amazon Inspector continually monitors resources. You can also use open source products such as OWASP Dependency-Check, Snyk, OpenVAS, package managers, and AWS Partner tools for vulnerability management. - Use AWS Systems Manager: You are responsible for patch management for your AWS resources, including Amazon Elastic Compute Cloud (Amazon EC2) instances, Amazon Machine Images (AMIs), and other compute resources. AWS Systems Manager Patch Manager automates the process of patching managed instances with both security related and other types of updates. Patch Manager can be used to apply patches on Amazon EC2 instances for both operating systems and applications, including Microsoft applications, Windows service packs, and minor version upgrades for Linux based instances. In addition to Amazon EC2, Patch Manager can also be used to patch on-premises servers. For a list of supported operating systems, see Supported operating systems in the Systems Manager User Guide. You can scan instances to see only a report of missing patches, or you can scan and automatically install all missing patches. - Use AWS Security Hub: Security Hub provides a comprehensive view of your security state in AWS. It collects security data across multiple AWS services and provides those findings in a standardized format, allowing you to prioritize security findings across AWS services. ## Infrastructure protection\\n\\n309\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Use AWS CloudFormation: AWS CloudFormation is an infrastructure as code (IaC) service that can help with vulnerability management by automating resource deployment and standardizing resource architecture across multiple accounts and environments. ### Resources\\n\\nRelated documents:\\n\\n- AWS Systems Manager\\n- Security Overview of AWS Lambda\\n- Amazon CodeGuru\\n- Improved, Automated Vulnerability Management for Cloud Workloads with a New Amazon Inspector\\n- Automate vulnerability management and remediation in AWS using Amazon Inspector and AWS Systems Manager – Part 1\\n\\nRelated videos:\\n\\n- Securing Serverless and Container Services\\n- Security best practices for the Amazon EC2 instance metadata service\\n\\nSEC06-BP02 Reduce attack surface\\n\\nReduce your exposure to unintended access by hardening operating systems and minimizing the components, libraries, and externally consumable services in use. Start by reducing unused components, whether they are operating system packages or applications, for Amazon Elastic Compute Cloud (Amazon EC2)-based workloads, or external software modules in your code, for all workloads. You can find many hardening and security configuration guides for common operating systems and server software. For example, you can start with the Center for Internet Security and iterate. In Amazon EC2, you can create your own Amazon Machine Images (AMIs), which you have patched and hardened, to help you meet the specific security requirements for your organization. The patches and other security controls you apply on the AMI are effective at the point in time in which they were created—they are not dynamic unless you modify after launching, for example, with AWS Systems Manager. ### Infrastructure protection\\n\\n310\\n---\\n## AWS Well-Architected Framework\\n\\nYou can simplify the process of building secure AMIs with EC2 Image Builder. EC2 Image Builder significantly reduces the effort required to create and maintain golden images without writing and maintaining automation. When software updates become available, Image Builder automatically produces a new image without requiring users to manually initiate image builds. EC2 Image Builder allows you to easily validate the functionality and security of your images before using them in production with AWS-provided tests and your own tests. You can also apply AWS-provided security settings to further secure your images to meet internal security criteria. For example, you can produce images that conform to the Security Technical Implementation Guide (STIG) standard using AWS-provided templates. Using third-party static code analysis tools, you can identify common security issues such as unchecked function input bounds, as well as applicable common vulnerabilities and exposures (CVEs). You can use Amazon CodeGuru for supported languages. Dependency checking tools can also be used to determine whether libraries your code links against are the latest versions, are themselves free of CVEs, and have licensing conditions that meet your software policy requirements. Using Amazon Inspector, you can perform configuration assessments against your instances for known CVEs, assess against security benchmarks, and automate the notification of defects. Amazon Inspector runs on production instances or in a build pipeline, and it notifies developers and engineers when findings are present. You can access findings programmatically and direct your team to backlogs and bug-tracking systems. EC2 Image Builder can be used to maintain server images (AMIs) with automated patching, AWS-provided security policy enforcement, and other customizations. When using containers implement ECR Image Scanning in your build pipeline and on a regular basis against your image repository to look for CVEs in your containers. While Amazon Inspector and other tools are effective at identifying configurations and any CVEs that are present, other methods are required to test your workload at the application level. Fuzzing is a well-known method of finding bugs using automation to inject malformed data into input fields and other areas of your application. Level of risk exposed if this best practice is not established: High\\n\\n### Implementation guidance\\n\\n- Harden operating system: Configure operating systems to meet best practices. ## Infrastructure protection\\n\\n311\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Harden containerized resources: Configure containerized resources to meet security best practices. - Implement AWS Lambda best practices. AWS Lambda best practices\\n\\n### Resources\\n\\nRelated documents:\\n\\n- AWS Systems Manager\\n- Replacing a Bastion Host with Amazon EC2 Systems Manager\\n- Security Overview of AWS Lambda\\n\\nRelated videos:\\n\\n- Running high-security workloads on Amazon EKS\\n- Securing Serverless and Container Services\\n- Security best practices for the Amazon EC2 instance metadata service\\n\\nRelated examples:\\n\\n- Lab: Automated Deployment of Web Application Firewall\\n\\nSEC06-BP03 Implement managed services\\n\\nImplement services that manage resources, such as Amazon Relational Database Service (Amazon RDS), AWS Lambda, and Amazon Elastic Container Service (Amazon ECS), to reduce your security maintenance tasks as part of the shared responsibility model. For example, Amazon RDS helps you set up, operate, and scale a relational database, automates administration tasks such as hardware provisioning, database setup, patching, and backups. This means you have more free time to focus on securing your application in other ways described in the AWS Well-Architected Framework. Lambda lets you run code without provisioning or managing servers, so you only need to focus on the connectivity, invocation, and security at the code level–not the infrastructure or operating system. Level of risk exposed if this best practice is not established: Medium\\n\\n### Infrastructure protection\\n\\n312\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n### Implementation guidance\\n\\n- Explore available services: Explore, test, and implement services that manage resources, such as Amazon RDS, AWS Lambda, and Amazon ECS. ### Resources\\n\\nRelated documents:\\n\\n- AWS Website\\n- AWS Systems Manager\\n- Replacing a Bastion Host with Amazon EC2 Systems Manager\\n- Security Overview of AWS Lambda\\n\\nRelated videos:\\n\\n- Running high-security workloads on Amazon EKS\\n- Securing Serverless and Container Services\\n- Security best practices for the Amazon EC2 instance metadata service\\n\\nRelated examples:\\n\\n- Lab: AWS Certificate Manager Request Public Certificate\\n\\nSEC06-BP04 Automate compute protection\\n\\nAutomate your protective compute mechanisms including vulnerability management, reduction in attack surface, and management of resources. The automation will help you invest time in securing other aspects of your workload, and reduce the risk of human error. Level of risk exposed if this best practice is not established: Medium\\n\\n### Implementation guidance\\n\\n- Automate configuration management: Enforce and validate secure configurations automatically by using a configuration management service or tool. - AWS Systems Manager\\n\\n## Infrastructure protection\\n\\n313\\n---\\n## AWS Well-Architected Framework\\n\\n- AWS CloudFormation\\n- Lab: Automated deployment of VPC\\n- Lab: Automated deployment of EC2 web application\\n\\nAutomate patching of Amazon Elastic Compute Cloud (Amazon EC2) instances: AWS Systems Manager Patch Manager automates the process of patching managed instances with both security-related and other types of updates. You can use Patch Manager to apply patches for both operating systems and applications. - AWS Systems Manager Patch Manager\\n- Centralized multi-account and multi-Region patching with AWS Systems Manager Automation\\n\\nImplement intrusion detection and prevention: Implement an intrusion detection and prevention tool to monitor and stop malicious activity on instances. Consider AWS Partner solutions: AWS Partners offer hundreds of industry-leading products that are equivalent, identical to, or integrate with existing controls in your on-premises environments. These products complement the existing AWS services to allow you to deploy a comprehensive security architecture and a more seamless experience across your cloud and on-premises environments. - Infrastructure security\\n\\n### Resources\\n\\nRelated documents:\\n\\n- AWS CloudFormation\\n- AWS Systems Manager\\n- AWS Systems Manager Patch Manager\\n- Centralized multi-account and multi-region patching with AWS Systems Manager Automation\\n- Infrastructure security\\n- Replacing a Bastion Host with Amazon EC2 Systems Manager\\n- Security Overview of AWS Lambda\\n\\nRelated videos:\\n\\nInfrastructure protection\\n314\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Running high-security workloads on Amazon EKS\\n- Securing Serverless and Container Services\\n- Security best practices for the Amazon EC2 instance metadata service\\n\\nRelated examples:\\n\\n- Lab: Automated Deployment of Web Application Firewall\\n- Lab: Automated deployment of Amazon EC2 web application\\n\\nSEC06-BP05 Enable people to perform actions at a distance\\n\\nRemoving the ability for interactive access reduces the risk of human error, and the potential for manual configuration or management. For example, use a change management workflow to deploy Amazon Elastic Compute Cloud (Amazon EC2) instances using infrastructure-as-code, then manage Amazon EC2 instances using tools such as AWS Systems Manager instead of allowing direct access or through a bastion host. AWS Systems Manager can automate a variety of maintenance and deployment tasks, using features including automation workflows, documents (playbooks), and the run command. AWS CloudFormation stacks build from pipelines and can automate your infrastructure deployment and management tasks without using the AWS Management Console or APIs directly. Level of risk exposed if this best practice is not established: Low\\n\\n### Implementation guidance\\n\\n- Replace console access: Replace console access (SSH or RDP) to instances with AWS Systems Manager Run Command to automate management tasks. - AWS Systems Manager Run Command\\n\\nResources\\n\\nRelated documents:\\n\\n- AWS Systems Manager\\n- AWS Systems Manager Run Command\\n- Replacing a Bastion Host with Amazon EC2 Systems Manager\\n\\nInfrastructure protection\\n\\n315\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\nSecurity Overview of AWS Lambda\\n\\nRelated videos:\\n\\n- Running high-security workloads on Amazon EKS\\n- Securing Serverless and Container Services\\n- Security best practices for the Amazon EC2 instance metadata service\\n\\nRelated examples:\\n\\n- Lab: Automated Deployment of Web Application Firewall\\n\\nSEC06-BP06 Validate software integrity\\n\\nImplement mechanisms (for example, code signing) to validate that the software, code and libraries used in the workload are from trusted sources and have not been tampered with. For example, you should verify the code signing certificate of binaries and scripts to confirm the author, and ensure it has not been tampered with since created by the author. AWS Signer can help ensure the trust and integrity of your code by centrally managing the code-signing lifecycle, including signing certification and public and private keys. You can learn how to use advanced patterns and best practices for code signing with AWS Lambda. Additionally, a checksum of software that you download, compared to that of the checksum from the provider, can help ensure it has not been tampered with. Level of risk exposed if this best practice is not established: Low\\n\\nImplementation guidance\\n\\n- Investigate mechanisms: Code signing is one mechanism that can be used to validate software integrity. - NIST: Security Considerations for Code Signing\\n\\nResources\\n\\nRelated documents:\\n\\n- AWS Signer\\n- New – Code Signing, a Trust and Integrity Control for AWS Lambda\\n\\nInfrastructure protection\\n\\n316\\n---\\n## AWS Well-Architected Framework\\n\\n### Data protection\\n\\n### Questions\\n\\nSEC 7. How do you classify your data?</DOCUMENT>\\n<DOCUMENT>After the user performs the specific tasks that required production access, access should be revoked. Limiting access to production environments helps prevent unintended production-impacting events and lowers the scope of impact of unintended access. - Consider permissions boundaries: A permissions boundary is a feature for using a managed policy that sets the maximum permissions that an identity-based policy can grant to an IAM entity. An entity's permissions boundary allows it to perform only the actions that are allowed by both its identity-based policies and its permissions boundaries. - Consider resource tags for permissions: An attribute-based access control model using resource tags allows you to grant access based on resource purpose, owner, environment, or other criteria. For example, you can use resource tags to differentiate between development and production environments. Using these tags, you can restrict developers to the development environment. By combining tagging and permissions policies, you can achieve fine-grained resource access without needing to define complicated, custom policies for every job function. - Use service control policies for AWS Organizations. Service control policies centrally control the maximum available permissions for member accounts in your organization. Importantly, service control policies allow you to restrict root user permissions in member accounts. Also consider using AWS Control Tower, which provides prescriptive managed controls that enrich AWS Organizations. You can also define your own controls within Control Tower. - Establish a user lifecycle policy for your organization: User lifecycle policies define tasks to perform when users are onboarded onto AWS, change job role or scope, or no longer need access to AWS. Permission reviews should be done during each step of a user’s lifecycle to verify that permissions are properly restrictive and to avoid permissions creep. - Establish a regular schedule to review permissions and remove any unneeded permissions: You should regularly review user access to verify that users do not have overly permissive access. AWS Config and IAM Access Analyzer can help when auditing user permissions. - Establish a job role matrix: A job role matrix visualizes the various roles and access levels required within your AWS footprint. Using a job role matrix, you can define and separate permissions based on user responsibilities within your organization. Use groups instead of applying permissions directly to individual users or roles. ## Identity and access management\\n\\n265\\n---\\n## AWS Well-Architected Framework\\n\\nResources\\n\\nRelated documents:\\n\\n- Grant least privilege\\n- Permissions boundaries for IAM entities\\n- Techniques for writing least privilege IAM policies\\n- IAM Access Analyzer makes it easier to implement least privilege permissions by generating IAM policies based on access activity\\n- Delegate permission management to developers by using IAM permissions boundaries\\n- Refining Permissions using last accessed information\\n- IAM policy types and when to use them\\n- Testing IAM policies with the IAM policy simulator\\n- Guardrails in AWS Control Tower\\n- Zero Trust architectures: An AWS perspective\\n- How to implement the principle of least privilege with CloudFormation StackSets\\n- Attribute-based access control (ABAC)\\n- Reducing policy scope by viewing user activity\\n- View role access\\n- Use Tagging to Organize Your Environment and Drive Accountability\\n- AWS Tagging Strategies\\n- Tagging AWS resources\\n\\nRelated videos:\\n\\n- Next-generation permissions management\\n- Zero Trust: An AWS perspective\\n\\nRelated examples:\\n\\n- Lab: IAM permissions boundaries delegating role creation\\n- Lab: IAM tag based access control for EC2\\n\\n## Identity and access management\\n\\n266\\n---\\n## AWS Well-Architected Framework\\n\\nSEC03-BP03 Establish emergency access process\\n\\nCreate a process that allows for emergency access to your workloads in the unlikely event of an issue with your centralized identity provider. You must design processes for different failure modes that may result in an emergency event. For example, under normal circumstances, your workforce users federate to the cloud using a centralized identity provider (SEC02-BP04) to manage their workloads. However, if your centralized identity provider fails, or the configuration for federation in the cloud is modified, then your workforce users may not be able to federate into the cloud. An emergency access process allows authorized administrators to access your cloud resources through alternate means (such as an alternate form of federation or direct user access) to fix issues with your federation configuration or your workloads. The emergency access process is used until the normal federation mechanism is restored. Desired outcome:\\n\\n- You have defined and documented the failure modes that count as an emergency: consider your normal circumstances and the systems your users depend on to manage their workloads. Consider how each of these dependencies can fail and cause an emergency situation. You may find the questions and best practices in the Reliability pillar useful to identify failure modes and architect more resilient systems to minimize the likelihood of failures. - You have documented the steps that must be followed to confirm a failure as an emergency. For example, you can require your identity administrators to check the status of your primary and standby identity providers and, if both are unavailable, declare an emergency event for identity provider failure. - You have defined an emergency access process specific to each type of emergency or failure mode. Being specific can reduce the temptation on the part of your users to overuse a general process for all types of emergencies. Your emergency access processes describe the circumstances under which each process should be used, and conversely situations where the process should not be used and points to alternate processes that may apply. - Your processes are well-documented with detailed instructions and playbooks that can be followed quickly and efficiently. Remember that an emergency event can be a stressful time for your users and they may be under extreme time pressure, so design your process to be as simple as possible. Common anti-patterns: Identity and access management\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- You do not have well-documented and well-tested emergency access processes. Your users are unprepared for an emergency and follow improvised processes when an emergency event arises. - Your emergency access processes depend on the same systems (such as a centralized identity provider) as your normal access mechanisms. This means that the failure of such a system may impact both your normal and emergency access mechanisms and impair your ability to recover from the failure. - Your emergency access processes are used in non-emergency situations. For example, your users frequently misuse emergency access processes as they find it easier to make changes directly than submit changes through a pipeline. - Your emergency access processes do not generate sufficient logs to audit the processes, or the logs are not monitored to alert for potential misuse of the processes. Benefits of establishing this best practice:\\n\\n- By having well-documented and well-tested emergency access processes, you can reduce the time taken by your users to respond to and resolve an emergency event. This can result in less downtime and higher availability of the services you provide to your customers. - You can track each emergency access request and detect and alert on unauthorized attempts to misuse the process for non-emergency events. Level of risk exposed if this best practice is not established: Medium\\n\\nImplementation guidance\\n\\nThis section provides guidance for creating emergency access processes for several failure modes related to workloads deployed on AWS, starting with common guidance that applies to all failure modes and followed by specific guidance based on the type of failure mode. Common guidance for all failure modes\\n\\nConsider the following as you design an emergency access process for a failure mode:\\n\\n- Document the pre-conditions and assumptions of the process: when the process should be used and when it should not be used. It helps to detail the failure mode and document assumptions, such as the state of other related systems. For example, the process for the Failure Mode 2 assumes that the identity provider is available, but the configuration on AWS is modified or has expired. ## Identity and access management\\n\\n268\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n- Pre-create resources needed by the emergency access process (SEC10-BP05). For example, pre-create the emergency access AWS account with IAM users and roles, and the cross-account IAM roles in all the workload accounts. This verifies that these resources are ready and available when an emergency event happens. By pre-creating resources, you do not have a dependency on AWS control plane APIs (used to create and modify AWS resources) that may be unavailable in an emergency. Further, by pre-creating IAM resources, you do not need to account for potential delays due to eventual consistency. - Include emergency access processes as part of your incident management plans (SEC10-BP02). Document how emergency events are tracked and communicated to others in your organization such as peer teams, your leadership, and, when applicable, externally to your customers and business partners. - Define the emergency access request process in your existing service request workflow system if you have one. Typically, such workflow systems allow you to create intake forms to collect information about the request, track the request through each stage of the workflow, and add both automated and manual approval steps. Relate each request with a corresponding emergency event tracked in your incident management system. Having a uniform system for emergency accesses allows you to track those requests in a single system, analyze usage trends, and improve your processes. - Verify that your emergency access processes can only be initiated by authorized users and require approvals from the user's peers or management as appropriate. The approval process should operate effectively both inside and outside business hours. Define how requests for approval allow secondary approvers if the primary approvers are unavailable and are escalated up your management chain until approved. - Verify that the process generates detailed audit logs and events for both successful and failed attempts to gain emergency access. Monitor both the request process and the emergency access mechanism to detect misuse or unauthorized accesses. Correlate activity with ongoing emergency events from your incident management system and alert when actions happen outside of expected time periods. For example, you should monitor and alert on activity in the emergency access AWS account, as it should never be used in normal operations. - Test emergency access processes periodically to verify that the steps are clear and grant the correct level of access quickly and efficiently. Your emergency access processes should be tested as part of incident response simulations (SEC10-BP07) and disaster recovery tests (REL13-BP03). Failure Mode 1: Identity provider used to federate to AWS is unavailable\\n\\nIdentity and access management\\n---\\n## AWS Well-Architected Framework\\n\\nAs described in SEC02-BP04 Rely on a centralized identity provider, we recommend relying on a centralized identity provider to federate your workforce users to grant access to AWS accounts. You can federate to multiple AWS accounts in your AWS organization using IAM Identity Center, or you can federate to individual AWS accounts using IAM. In both cases, workforce users authenticate with your centralized identity provider before being redirected to an AWS sign-in endpoint to single sign-on. In the unlikely event that your centralized identity provider is unavailable, your workforce users can't federate to AWS accounts or manage their workloads. In this emergency event, you can provide an emergency access process for a small set of administrators to access AWS accounts to perform critical tasks that cannot wait until your centralized identity providers are back online. For example, your identity provider is unavailable for 4 hours and during that period you need to modify the upper limits of an Amazon EC2 Auto Scaling group in a Production account to handle an unexpected spike in customer traffic. Your emergency administrators should follow the emergency access process to gain access to the specific production AWS account and make the necessary changes. The emergency access process relies on a pre-created emergency access AWS account that is used solely for emergency access and has AWS resources (such as IAM roles and IAM users) to support the emergency access process. During normal operations, no one should access the emergency access account and you must monitor and alert on the misuse of this account (for more detail, see the preceding Common guidance section). The emergency access account has emergency access IAM roles with permissions to assume cross-account roles in the AWS accounts that require emergency access. These IAM roles are pre-created and configured with trust policies that trust the emergency account's IAM roles. The emergency access process can use one of the following approaches:\\n\\n- You can pre-create a set of IAM users for your emergency administrators in the emergency access account with associated strong passwords and MFA tokens. These IAM users have permissions to assume the IAM roles that then allow cross-account access to the AWS account where emergency access is required. We recommend creating as few such users as possible and assigning each user to a single emergency administrator. During an emergency, an emergency administrator user signs into the emergency access account using their password and MFA token code, switches to the emergency access IAM role in the emergency account, and finally switches to the emergency access IAM role in the workload account to perform the emergency change action. The advantage of this approach is that each IAM user is assigned to one emergency administrator and you can know which user signed-in by reviewing CloudTrail events. The disadvantage is that\\n\\n## Identity and access management\\n\\n270\\n---\\n## AWS Well-Architected Framework\\n\\nyou have to maintain multiple IAM users with their associated long-lived passwords and MFA tokens. You can use the emergency access AWS account root user to sign into the emergency access account, assume the IAM role for emergency access, and assume the cross-account role in the workload account. We recommend setting a strong password and multiple MFA tokens for the root user. We also recommend storing the password and the MFA tokens in a secure enterprise credential vault that enforces strong authentication and authorization. You should secure the password and MFA token reset factors: set the email address for the account to an email distribution list that is monitored by your cloud security administrators, and the phone number of the account to a shared phone number that is also monitored by security administrators. The advantage of this approach is that there is one set of root user credentials to manage. The disadvantage is that since this is a shared user, multiple administrators have the ability to sign in as the root user. You must audit your enterprise vault log events to identify which administrator checked out the root user password. ### Failure Mode 2: Identity provider configuration on AWS is modified or has expired\\n\\nTo allow your workforce users to federate to AWS accounts, you can configure the IAM Identity Center with an external identity provider or create an IAM Identity Provider (SEC02-BP04). Typically, you configure these by importing a SAML meta-data XML document provided by your identity provider. The meta-data XML document includes a X.509 certificate corresponding to a private key that the identity provider uses to sign its SAML assertions. These configurations on the AWS-side may be modified or deleted by mistake by an administrator. In another scenario, the X.509 certificate imported into AWS may expire and a new meta-data XML with a new certificate has not yet been imported into AWS. Both scenarios can break federation to AWS for your workforce users, resulting in an emergency. In such an emergency event, you can provide your identity administrators access to AWS to fix the federation issues. For example, your identity administrator uses the emergency access process to sign into the emergency access AWS account, switches to a role in the Identity Center administrator account, and updates the external identity provider configuration by importing the latest SAML meta-data XML document from your identity provider to re-enable federation. Once federation is fixed, your workforce users continue to use the normal operating process to federate into their workload accounts. You can follow the approaches detailed in the previous Failure Mode 1 to create an emergency access process. You can grant least-privilege permissions to your identity administrators to access Identity and access management. ---\\n## AWS Well-Architected Framework\\n\\nonly the Identity Center administrator account and perform actions on Identity Center in that account. ### Failure Mode 3: Identity Center disruption\\n\\nIn the unlikely event of an IAM Identity Center or AWS Region disruption, we recommend that you set up a configuration that you can use to provide temporary access to the AWS Management Console. The emergency access process uses direct federation from your identity provider to IAM in an emergency account. For detail on the process and design considerations, see Set up emergency access to the AWS Management Console. ### Implementation steps\\n\\nCommon steps for all failure modes\\n\\n- Create an AWS account dedicated to emergency access processes. Pre-create the IAM resources needed in the account such as IAM roles or IAM users, and optionally IAM Identity Providers. Additionally, pre-create cross-account IAM roles in the workload AWS accounts with trust relationships with corresponding IAM roles in the emergency access account. You can use AWS CloudFormation StackSets with AWS Organizations to create such resources in the member accounts in your organization. - Create AWS Organizations service control policies (SCPs) to deny the deletion and modification of the cross-account IAM roles in the member AWS accounts. - Enable CloudTrail for the emergency access AWS account and send the trail events to a central S3 bucket in your log collection AWS account. If you are using AWS Control Tower to set up and govern your AWS multi-account environment, then every account you create using AWS Control Tower or enroll in AWS Control Tower has CloudTrail enabled by default and sent to an S3 bucket in a dedicated log archive AWS account. - Monitor the emergency access account for activity by creating EventBridge rules that match on console login and API activity by the emergency IAM roles. Send notifications to your security operations center when activity happens outside of an ongoing emergency event tracked in your incident management system. Additional steps for Failure Mode 1: Identity provider used to federate to AWS is unavailable and Failure Mode 2: Identity provider configuration on AWS is modified or has expired\\n\\n### Identity and access management\\n\\n272\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n### Pre-create resources depending on the mechanism you choose for emergency access:\\n\\n|Using IAM users:|pre-create the IAM users with strong passwords and associated MFA devices.|\\n|---|---|\\n|Using the emergency account root user:|configure the root user with a strong password and store the password in your enterprise credential vault. Associate multiple physical MFA devices with the root user and store the devices in locations that can be accessed quickly by members of your emergency administrator team.|\\n\\n### Additional steps for Failure Mode 3: Identity center disruption\\n\\n- As detailed in Set up emergency access to the AWS Management Console, in the emergency access AWS account, create an IAM Identity Provider to enable direct SAML federation from your identity provider. - Create emergency operations groups in your IdP with no members.</DOCUMENT>\", 'question': 'What does SUS02-BP01 recommend for managing workload infrastructure?'}\n",
      "BadRequestError('Error code: 400 - {\\'object\\': \\'error\\', \\'message\\': \"This model\\'s maximum context length is 8192 tokens. However, you requested 8562 tokens (8062 in the messages, 500 in the completion). Please reduce the length of the messages or completion.\", \\'type\\': \\'BadRequestError\\', \\'param\\': None, \\'code\\': 400}')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[------------------------------------------------->] 30/30gpt-3.5-turbo-0125\n",
      "View the evaluation results for project 'Evaluation - gpt-3.5-turbo-0125' at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644/compare?selectedSessions=b34e329c-ab5f-4409-8fad-aae3750d4db1\n",
      "\n",
      "View all tests for Dataset RAFT at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644\n",
      "[------------------------------------------------->] 30/30claude-3-haiku-20240307\n",
      "View the evaluation results for project 'Evaluation - claude-3-haiku-20240307' at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644/compare?selectedSessions=2d555b66-90c2-47c7-877d-6785b79806d3\n",
      "\n",
      "View all tests for Dataset RAFT at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644\n",
      "[>                                                 ] 0/30"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Chain failed for example 4d69f923-db86-4f23-bda7-01926c92f9ed with inputs {'answer': '##Reason: The document provides a clear recommendation for compiling source code and running unit tests within the AWS ecosystem. Specifically, it states ##begin_quote## Use CodeBuild to compile your source code, runs unit tests, and produces artifacts that are ready to deploy. ##end_quote## This indicates that AWS CodeBuild is the recommended service for these tasks. ##Answer: AWS CodeBuild', 'context': \"Carefully read and analyze the provided documents to answer the question that follows. Provide a detailed, step-by-step explanation of your reasoning, demonstrating how you arrived at your conclusion based on the information given in the documents.\\n\\n<DOCUMENT>This reduces lead time, decreases cost, encourages increased frequency of change, reduces the level of effort, and increases collaboration. Prepare\\n\\n127\\n---\\n## AWS Well-Architected Framework\\n\\n### Implementation steps\\n\\n|Author|Source/Artifact|Build & test|Deploy|Monitor|\\n|---|---|---|---|---|\\n|AWS Cloud|AWS|AWS CodeBuild|AWS|AWS Amazona Ammazom|\\n|AWS Toolkits|Codecommit| |Loocucpiov|X-Ray CloudWatch DevOps|\\n| |Amazon ECR & Amazom| | |AWS Amazon Amazon|\\n| |Amazon ECR CodeGuru| | |Config Managed Managed|\\n| |Public| | |Grldna Service for Prometheus|\\n| | |AWS Cloud Development Kit|AWS Serverless|Appllcatlon Model|\\n|Model|AWS CloudFormation|CDK for Kubernetes| | |\\n| |AWS CDK for Terraform| | | |\\n\\nDiagram showing a CI/CD pipeline using AWS CodePipeline and related services\\n\\n1. Use AWS CodeCommit to version control, store, and manage assets (such as documents, source code, and binary files). 2. Use CodeBuild to compile your source code, runs unit tests, and produces artifacts that are ready to deploy. 3. Use CodeDeploy as a deployment service that automates application deployments to Amazon EC2 instances, on-premises instances, serverless AWS Lambda functions, or Amazon ECS. 4.</DOCUMENT>\\n<DOCUMENT>4. Enable X-Ray Insights:\\n- Turn on X-Ray Insights for automated anomaly detection in traces. - Examine insights to pinpoint patterns and ascertain root causes, such as increased fault rates or latencies. - Consult the insights timeline for a chronological analysis of detected issues. Operate 177\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\n|5. Use X-Ray Analytics:|X-Ray Analytics allows you to thoroughly explore trace data, pinpoint patterns, and extract insights.|\\n|---|---|\\n|6. Use groups in X-Ray:|Create groups in X-Ray to filter traces based on criteria such as high latency, allowing for more targeted analysis.|\\n|7. Incorporate Amazon DevOps Guru:|Engage Amazon DevOps Guru to benefit from machine learning models pinpointing operational anomalies in traces.|\\n|8. Use CloudWatch Synthetics:|Use CloudWatch Synthetics to create canaries for continually monitoring your endpoints and workflows. These canaries can integrate with X-Ray to provide trace data for in-depth analysis of the applications being tested.|\\n|9. Use Real User Monitoring (RUM):|With AWS X-Ray and CloudWatch RUM, you can analyze and debug the request path starting from end users of your application through downstream AWS managed services. This helps you identify latency trends and errors that impact your users.|\\n|10. Correlate with logs:|Correlate trace data with related logs within the X-Ray trace view for a granular perspective on application behavior. This allows you to view log events directly associated with traced transactions.|\\n\\nLevel of effort for the implementation plan: Medium\\n\\n### Resources\\n\\nRelated best practices:\\n\\n- OPS08-BP01 Analyze workload metrics\\n- OPS08-BP02 Analyze workload logs\\n\\nRelated documents:\\n\\n- Using ServiceLens to Monitor Application Health\\n- Exploring Trace Data with X-Ray Analytics\\n- Detecting Anomalies in Traces with X-Ray Insights\\n- Continuous Monitoring with CloudWatch Synthetics\\n\\nRelated videos:\\n\\n- Analyze and Debug Applications Using Amazon CloudWatch Synthetics and AWS X-Ray\\n\\nOperate 178\\n---\\n## AWS Well-Architected Framework\\n\\nFramework\\n\\nUse AWS X-Ray Insights\\n\\nRelated examples:\\n\\n- One Observability Workshop\\n- Implementing X-Ray with AWS Lambda\\n- CloudWatch Synthetics Canary Templates\\n\\nOPS08-BP04 Create actionable alerts\\n\\nPromptly detecting and responding to deviations in your application's behavior is crucial. Especially vital is recognizing when outcomes based on key performance indicators (KPIs) are at risk or when unexpected anomalies arise.</DOCUMENT>\\n<DOCUMENT>Services should also protect themselves from abnormally expensive content with throttles and server-side timeouts. Requests that take abnormally long due to a service impairment can be timed out and retried. Consideration should be given to service costs for the request and retry, but if the cause is a localized impairment, a retry is not likely to be expensive and will reduce client resource consumption. The timeout may also release server resources depending on the nature of the impairment. Requests that take a long time to complete because the request or response has failed to be delivered by the network can be timed out and retried. Because the request or response was not delivered, failure would have been the outcome regardless of the length of timeout. Timing out in this case will not release server resources, but it will release client resources and improve workload performance. Take advantage of well-established design patterns like retries and circuit breakers to handle timeouts gracefully and support fail-fast approaches. AWS SDKs and AWS CLI allow for configuration of both connection and request timeouts and for retries with exponential backoff and jitter. AWS Lambda functions support configuration of timeouts, and with AWS Step Functions, you can build low code circuit breakers that take advantage of pre-built integrations with AWS services and SDKs. AWS App Mesh Envoy provides timeout and circuit breaker capabilities. ## Implementation steps\\n\\n- Configure timeouts on remote service calls and take advantage of built-in language timeout features or open source timeout libraries. - When your workload makes calls with an AWS SDK, review the documentation for language specific timeout configuration. ## Workload architecture\\n\\n459\\n---\\n## AWS Well-Architected Framework\\n\\n- C++\\n\\nWhen using AWS SDKs or AWS CLI commands in your workload, configure default timeout values by setting the AWS configuration defaults for connectTimeoutInMillis and tlsNegotiationTimeoutInMillis. Apply command line options cli-connect-timeout and cli-read-timeout to control one-off AWS CLI commands to AWS services. Monitor remote service calls for timeouts, and set alarms on persistent errors so that you can proactively handle error scenarios. Implement CloudWatch Metrics and CloudWatch anomaly detection on call error rates, service level objectives for latency, and latency outliers to provide insight into managing overly aggressive or permissive timeouts. Configure timeouts on Lambda functions. API Gateway clients must implement their own retries when handling timeouts. API Gateway supports a 50 millisecond to 29 second integration timeout for downstream integrations and does not retry when integration requests timeout. Implement the circuit breaker pattern to avoid making remote calls when they are timing out. Open the circuit to avoid failing calls and close the circuit when calls are responding normally. For container based workloads, review App Mesh Envoy features to leverage built-in timeouts and circuit breakers.</DOCUMENT>\\n<DOCUMENT>How do you manage demand, and supply resources? COST 9. How do you manage demand, and supply resources? For a workload that has balanced spend and performance, verify that everything you pay for is used and avoid significantly underutilizing instances. A skewed utilization metric in either direction has an adverse impact on your organization, in either operational costs (degraded performance due to over-utilization), or wasted AWS expenditures (due to over-provisioning). ### Best practices\\n\\n- COST09-BP01 Perform an analysis on the workload demand\\n- COST09-BP02 Implement a buffer or throttle to manage demand\\n- COST09-BP03 Supply resources dynamically\\n\\nCOST09-BP01 Perform an analysis on the workload demand\\n\\nAnalyze the demand of the workload over time. Verify that the analysis covers seasonal trends and accurately represents operating conditions over the full workload lifetime. Analysis effort should reflect the potential benefit, for example, time spent is proportional to the workload cost. Level of risk exposed if this best practice is not established: High\\n\\nImplementation guidance\\n\\nAnalyzing workload demand for cloud computing involves understanding the patterns and characteristics of computing tasks that are initiated in the cloud environment. This analysis helps users optimize resource allocation, manage costs, and verify that performance meets required levels. ## Manage demand and supply resources\\n\\n795\\n---\\n## AWS Well-Architected Framework\\n\\nKnow the requirements of the workload. Your organization's requirements should indicate the workload response times for requests. The response time can be used to determine if the demand is managed, or if the supply of resources should change to meet the demand. The analysis should include the predictability and repeatability of the demand, the rate of change in demand, and the amount of change in demand. Perform the analysis over a long enough period to incorporate any seasonal variance, such as end-of-month processing or holiday peaks. Analysis effort should reflect the potential benefits of implementing scaling. Look at the expected total cost of the component and any increases or decreases in usage and cost over the workload's lifetime. The following are some key aspects to consider when performing workload demand analysis for cloud computing:\\n\\n|1. Resource utilization and performance metrics:|Analyze how AWS resources are being used over time. Determine peak and off-peak usage patterns to optimize resource allocation and scaling strategies. Monitor performance metrics such as response times, latency, throughput, and error rates. These metrics help assess the overall health and efficiency of the cloud infrastructure.|\\n|---|---|\\n|2. User and application scaling behaviour:|Understand user behavior and how it affects workload demand. Examining the patterns of user traffic assists in enhancing the delivery of content and the responsiveness of applications. Analyze how workloads scale with increasing demand. Determine whether auto-scaling parameters are configured correctly and effectively for handling load fluctuations.|\\n|3. Workload types:|Identify the different types of workloads running in the cloud, such as batch processing, real-time data processing, web applications, databases, or machine learning. Each type of workload may have different resource requirements and performance profiles.|\\n|4. Service-level agreements (SLAs):|Compare actual performance with SLAs to ensure compliance and identify areas that need improvement.|\\n\\nYou can use Amazon CloudWatch to collect and track metrics, monitor log files, set alarms, and automatically react to changes in your AWS resources. You can also use Amazon CloudWatch to gain system-wide visibility into resource utilization, application performance, and operational health. With AWS Trusted Advisor, you can provision your resources following best practices to improve system performance and reliability, increase security, and look for opportunities to save money. Manage demand and supply resources\\n---\\n## AWS Well-Architected Framework\\n\\nYou can also turn off non-production instances and use Amazon CloudWatch and Auto Scaling to match increases or reductions in demand. Finally, you can use AWS Cost Explorer or Amazon QuickSight with the AWS Cost and Usage Report (CUR) file or your application logs to perform advanced analysis of workload demand. Overall, a comprehensive workload demand analysis allows organizations to make informed decisions about resource provisioning, scaling, and optimization, leading to better performance, cost efficiency, and user satisfaction. ### Implementation steps\\n\\n- Analyze existing workload data: Analyze data from the existing workload, previous versions of the workload, or predicted usage patterns. Use Amazon CloudWatch, log files and monitoring data to gain insight on how workload was used. Analyze a full cycle of the workload, and collect data for any seasonal changes such as end-of-month or end-of-year events. The effort reflected in the analysis should reflect the workload characteristics. The largest effort should be placed on high-value workloads that have the largest changes in demand. The least effort should be placed on low-value workloads that have minimal changes in demand. - Forecast outside influence: Meet with team members from across the organization that can influence or change the demand in the workload. Common teams would be sales, marketing, or business development. Work with them to know the cycles they operate within, and if there are any events that would change the demand of the workload.</DOCUMENT>\", 'question': 'Which AWS service is recommended for compiling source code and running unit tests?'}\n",
      "RateLimitError(\"Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[------------------------------------------------->] 30/30"
     ]
    }
   ],
   "source": [
    "for each in [raft_starling_7b, openai_llm, anthropic_llm]:\n",
    "    \n",
    "    MODEL_NAME = each.model_name if hasattr(each, \"model_name\") else each.model \n",
    "    print(MODEL_NAME)\n",
    "\n",
    "    await arun_on_dataset(\n",
    "        dataset_name=dataset_name,\n",
    "        llm_or_chain_factory=create_chain_by_model(model=each),\n",
    "        client=langsmith_client,\n",
    "        evaluation=evaluation_config,\n",
    "        project_name=f\"Evaluation - {MODEL_NAME}\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Evaluating Without context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define chain\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    messages=[(\"system\", \"You are an expert QA assistant, answer the following question\"), (\"user\", \"{question}\")]\n",
    ")\n",
    "\n",
    "output_parser = StrOutputParser()\n",
    "\n",
    "def create_chain_by_model(model):\n",
    "    return (\n",
    "        prompt \n",
    "        | model \n",
    "        | output_parser\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define metrics of eval\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        \"cot_qa\",\n",
    "        RunEvalConfig.LabeledCriteria(\n",
    "            criteria=\"helpfulness\",\n",
    "            input_key=\"question\",\n",
    "            prediction_key=\"output\",\n",
    "        ),\n",
    "        RunEvalConfig.LabeledCriteria(\n",
    "            criteria=\"coherence\",\n",
    "            input_key=\"question\",\n",
    "            prediction_key=\"output\",\n",
    "        )\n",
    "    ],\n",
    "    input_key=\"question\",\n",
    "    eval_llm=eval_llm,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jjovalle99/starling-7b-raft-ft\n",
      "View the evaluation results for project 'WC - Evaluation - jjovalle99/starling-7b-raft-ft' at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644/compare?selectedSessions=56300fb5-7d29-4044-b650-ed5fbef6f099\n",
      "\n",
      "View all tests for Dataset RAFT at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644\n",
      "[------------------------------------------------->] 30/30gpt-3.5-turbo-0125\n",
      "View the evaluation results for project 'WC - Evaluation - gpt-3.5-turbo-0125' at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644/compare?selectedSessions=55698c93-89ad-4c0a-9dd1-8575fc7487e6\n",
      "\n",
      "View all tests for Dataset RAFT at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644\n",
      "[------------------------------------------------->] 30/30claude-3-haiku-20240307\n",
      "View the evaluation results for project 'WC - Evaluation - claude-3-haiku-20240307' at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644/compare?selectedSessions=fb1129ab-c8d5-4cbb-8320-619e58197e2c\n",
      "\n",
      "View all tests for Dataset RAFT at:\n",
      "https://smith.langchain.com/o/e1ff5e9a-fc1a-5ec0-91dc-86daf509e790/datasets/9dd76add-d39c-4897-8b54-d762412f1644\n",
      "[------------------------------------------------->] 30/30"
     ]
    }
   ],
   "source": [
    "for each in [raft_starling_7b, openai_llm, anthropic_llm]:\n",
    "    \n",
    "    MODEL_NAME = each.model_name if hasattr(each, \"model_name\") else each.model \n",
    "    print(MODEL_NAME)\n",
    "\n",
    "    await arun_on_dataset(\n",
    "        dataset_name=dataset_name,\n",
    "        llm_or_chain_factory=create_chain_by_model(model=each),\n",
    "        client=langsmith_client,\n",
    "        evaluation=evaluation_config,\n",
    "        concurrency_level=10 if \"haiku\" not in MODEL_NAME else 2,\n",
    "        project_name=f\"WC - Evaluation - {MODEL_NAME}\",\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}