o Jjg@s dZddlmZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZdd lmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l"m&Z'dd l"m(Z)ddl*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6ddl4m7Z8ddl9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKddlLmMZMmNZNmOZOddlPmQZRdd lSm&ZTddlSmUZUmVZVe rddlWZXeYeZZ[eegeeIe#ffeee\gefe#eIfZ]eegeeIe#ffefZ^Gddde_Z`Gd d!d!e\ZaGd"d#d#e\Zb $ddd+d,Zcdd/d0ZdGd1d2d2eGZedd4d5Zfdd;d<Zgdd?d@ZhddBdCZiddKdLZjddRdSZkddUdVZlddXdYZmddcddZnddfdgZoddidjZpdddddkddtduZqddddvddydzZrdd{dd~dZsddddvdddZtddddvdddZudd{dddZv   ddddZwGdddeGddZxejyGdddZzdddZ{dddZ|dZ}dddddddddddZ~dddddddddddZdZee_edde~_dS)z>Utilities for running language models or Chains over datasets.) annotationsN)datetimetimezone) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnioncast)warn_deprecated) Callbacks)BaseLanguageModel) BaseMessagemessages_from_dict) ChatResult LLMResult)RunnableRunnableConfigRunnableLambdaconfig)utils)EvaluatorCallbackHandlerwait_for_all_evaluators)LangChainTracer)Client) get_git_infoget_langchain_env_var_metadata)EvaluationResult RunEvaluator) run_evaluator) as_runnableis_traceable_function)DatasetDataTypeExampleRun TracerSession)LangSmithError) HTTPError) TypedDict)Chain)load_evaluator) EvaluatorTypePairwiseStringEvaluatorStringEvaluator) evaluation)name_generationprogressc@seZdZdZdS)InputFormatErrorz(Raised when the input format is invalid.N)__name__ __module__ __qualname____doc__r;r;_/var/www/html/zoom/venv/lib/python3.10/site-packages/langchain/smith/evaluation/runner_utils.pyr6Nsr6c@s$eZdZdZd ddZd ddZdS) TestResultz1A dictionary of the results of a single test run.return pd.DataFramecCs.|}dd|jD}|jddj|ddS)zReturn quantiles for the feedback scores. This method calculates and prints the quantiles for the feedback scores across all feedback keys. Returns: A DataFrame containing the quantiles for each feedback key. cSs6g|]}|ds|ds|dvs|dr|qS)inputs.outputs.>inputoutput reference) startswith).0colr;r;r< esz5TestResult.get_aggregate_feedback..all)include)axis) to_dataframecolumnsdescribedrop)selfdfto_dropr;r;r<get_aggregate_feedbackXs  z!TestResult.get_aggregate_feedbackc CsHzddl}Wnty}ztd|d}~wwg}g}|dD]z\}}|d}|d}t|tr>dd|D} n |durEi} nd|i} id d|d D| } d |vrxt|d trr| d d|d Dn|d | d <| id d|D|d|d|dd|| ||q"|j||dS)z#Convert the results to a dataframe.rNzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.resultsfeedbackrCcSi|] \}}d||qS)rAr;rFkvr;r;r< z+TestResult.to_dataframe..cSrW)r@r;rXr;r;r<r[r\rBrDcSrW)z reference.r;rXr;r;r<r[r\cSsi|] }d|j|jqS)z feedback.)keyscore)rFfr;r;r<r[r\Errorexecution_timerun_id)errorrarb)index) pandas ImportErroritemsget isinstancedictupdateappend DataFrame) rQpdeindicesrecords example_idresultrVoutput_rCrr;r;r<rMosV       zTestResult.to_dataframeN)r>r?)r7r8r9r:rTrMr;r;r;r<r=Us r=cs,eZdZdZdfdd Zdd d ZZS) EvalErrorz"Your architecture raised an error.r` BaseExceptionkwargsrr>Nonec stjdd|i|dS)Nr`r;)super__init__)rQr`rx __class__r;r<r{szEvalError.__init__namestrcCs*z||WStytd|dw)Nz%'EvalError' object has no attribute '')KeyErrorAttributeError)rQr~r;r;r< __getattr__s   zEvalError.__getattr__)r`rwrxrr>ry)r~rr>r)r7r8r9r:r{r __classcell__r;r;r|r<rvsrv llm_or_chain_factoryMODEL_OR_CHAIN_FACTORY dataset_namerr>MCFcsZt|tr)|jj}|jdur#jjj}td|d|d|dfddSt|tr0|St|tr=|fddSt|rt |rRt t t |fd dSz|}Wn&t y}t t |}t|}td |d t|fd dYSwt t |t|tr|St t t |rt t t |fd dSt|tsfddSS|S)zForgive the user if they pass in a chain without memory instead of a chain factory. It's a common mistake. Raise a more helpful error message as well.Na$Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called. This will safegaurd against information leakage between dataset examples. For example: def chain_constructor(): new_memory = z(...) return z*(memory=new_memory, ...) run_on_dataset("z", chain_constructor, ...)cSNr;r;)chainr;r<z(_wrap_in_chain_factory..crrr;r;)lcfr;r<rrcrrr;r; runnable_r;r<rrzWrapping function z as RunnableLambda.crrr;r;)wrappedr;r<rrcrrr;r;rr;r<rrcstSr)rr;) constructorr;r<r)rir.r}r7memory ValueErrorrrcallabler%r$r r TypeErrorinspect signatureloggerinfor)rr chain_class memory_class_model user_funcsigr;)rrrrrr<_wrap_in_chain_factorysV                 rinputsDict[str, Any]cCs,|stdg}d|vr%t|dtstdt|dj|dg}n]d|vrLt|dtr;tdd|dDsGtdt|dj|d}n6t|dkr{tt | }t|trc|g}nt|trttd d|Drt|}ntd |td |t|dkr|d Std t|d)zGet prompt from inputs. Args: inputs: The input dictionary. Returns: A string prompt. Raises: InputFormatError: If the input format is invalid. Inputs should not be empty.promptz"Expected string for 'prompt', got promptscs|]}t|tVqdSrrirrFir;r;r<   z_get_prompt..z,Expected list of strings for 'prompts', got rKcsrrrrr;r;r<r z)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got rz)LLM Run expects single prompt input. Got z prompts.) r6rirtyper7listrIlennextitervalues)rrprompt_r;r;r< _get_promptsH        rc@seZdZUdZded<dS)ChatModelInputzVInput for a chat model. Parameters: messages: List of chat messages. zList[BaseMessage]messagesNr7r8r9r:__annotations__r;r;r;r<rs  rrjcCs|std|}d|vr|d|d<nt|dkr&tt||d<d|vrS|d}t|tr?t dd|Dr?|g}t|dkrOt |d|d<|Stdtd |) zGet Chat Messages from inputs. Args: inputs: The input dictionary. Returns: A list of chat messages. Raises: InputFormatError: If the input format is invalid. rrrBrKcsrr)rirjrr;r;r<r9rz _get_messages..rzGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got ) r6copypoprrrrrirrIr)r input_copy raw_messagesr;r;r< _get_messages%s0   r first_exampler( input_mapperOptional[Callable[[Dict], Any]]ryc Cs|r+||j}t|ts't|trtdd|Ds)td|dt|ddSdSzt|jWdStyTz t|jWYdStyStd|jdww)NcsrrrirrFmsgr;r;r<rUrz>_validate_example_inputs_for_language_model..zWhen using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages. Got: of type .zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.) rrirrrIr6rrr)rr prompt_inputr;r;r<+_validate_example_inputs_for_language_modelMs8     rrr.cCs|r2||j}t|j|}t|ts!td|dt|d|r0td|jd|dS|j}t|j|}t |dkrLt |jdkrLdS|r[td|jd|dS) zt|trd\}}d}nd}|}t|tr|jnd}t|tr%|jnd}t||||djr7t|djnd||}|Sd}|S)zt|tr|St|ttfr!t|tst|}t||d}|j} nBt|tjrRd|i|} t|j fi| }|j j} t|tj rQ|j pF|}|j pK|}|j pP|}nt|rZt|Stdt|t|tr|jrz|durztd| d|dtjj||||||| gd} | St|trtd| d td| d ) N)rrzUnknown evaluator type: zPMust specify reference_key in smith_eval.RunEvalConfig to use evaluator of type z) with dataset with multiple output keys: r)rrrtagszRun evaluator for z is not implemented. PairwiseStringEvaluators compare the outputs of two different models rather than the output of a single model. Did you mean to use a StringEvaluator instead? See: https://python.langchain.com/docs/guides/evaluation/string/z is not implemented)rir"r0rr/valuesmith_eval_config EvalConfig get_kwargsevaluator_typeSingleKeyEvalConfigrrrrrun_evaluator_decrrr2requires_reference smith_evalStringRunEvaluatorChainfrom_run_and_data_typer1NotImplementedError) rrrrrrrr evaluator_ eval_type_tagrxr#r;r;r<_construct_run_evaluatorsZ            r2Tuple[Optional[str], Optional[str], Optional[str]]cCs(t||}t||}t||}|||fSr)rrr)rrrrrrrr;r;r< _get_keysIs    rList[RunEvaluator]c Csg}d\}}} |js|jr!tdd|jDr!t||||\}}} |jD]} t| |j|||| ||} || q$|jp.)rrrzUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.) evaluatorscustom_evaluatorsanyrrrrlrir"r2rrrrrr)rrrrrrrrrrrr#rcustom_evaluatorr;r;r<rUsV        rr callbacksrmetadatarrrr rr Optional[Dict[str, Any]]Union[str, BaseMessage]c s|dur7||}t|tst|tr/tdd|Dr/|j|t||p$g|p'iddIdHStd|dzt|}|j|t||pDg|pGiddIdH}W|Stywt|} |jd i| dt||phg|pkidiIdH}Y|Sw) aAsynchronously run the language model. Args: llm: The language model to run. inputs: The input dictionary. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map inputs to the expected format. Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Ncsrrrrr;r;r<rrz_arun_llm..r rr rz%Input mapper returned invalid format 3 Expected a single string or list of chat messages.rr;) rirrrIainvokerr6rr) rrrr rr prompt_or_messagesr llm_output llm_inputsr;r;r< _arun_llmsP  rrrr Union[Chain, Runnable]Union[dict, str]c s|dur|n||}t|tr;t|tr;t|dkr;|jr;tt|}|j|t ||p.g|p1iddIdH}|St |p?g||pCid} |j|| dIdH}|S)z%Run a chain asynchronously on inputs.NrKrrrr r ) rir.rjrrrrrrr rrr rrr inputs_valrCrunnable_configr;r;r< _arun_chains,    r)rr'Union[dict, str, LLMResult, ChatResult]c st|trdnd}d}z8t|tr(t||j|d|d||ddIdH}n|}t||j|d|d||ddIdH}|}W|Styp}z t|d|j d |jd t |t |d }WYd}~|Sd}~ww) aAsynchronously run the Chain or language model. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map the input to the expected format. Returns: A list of outputs. LLMr.Nrr r r  failed for example with inputs  r`) rirrrrhr Exceptionrridreprrv) rrrr chain_or_llmrsrCrror;r;r<_arun_llm_or_chainsJ   r*c Cs|dur5||}t|tst|tr-tdd|Dr-|j|t||p#g|p&idd}|Std|dzt|}|j|t||pBg|pEidd}W|Stylt|} |jd i| dt||pcid i}Y|Sw) a Run the language model on the example. Args: llm: The language model to run. inputs: The input dictionary. callbacks: The callbacks to use during the run. tags: Optional tags to add to the run. input_mapper: function to map to the inputs dictionary from an Example Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Ncsrrrrr;r;r<rOrz_run_llm..rrz'Input mapper returned invalid format: rr)r r r;) rirrrIinvokerr6rr) rrr rrr rr llm_promptsrr;r;r<_run_llm1sL  r-Union[Dict, str]c Cs|dur|n||}t|tr7t|tr7t|dkr7|jr7tt|}|j|t ||p-g|p0idd}|St |p;g||p?id} |j|| d}|S)zRun a chain on inputs.NrKrrr) rir.rjrrrrrr+rrr;r;r< _run_chainos*   r/c Cst|trdnd}d}z2t|tr$t||j|d|d||dd}n|}t||j|d|d||dd}|}W|Styo}z&t|j} t |d|j d |jd | d | t |d }WYd}~|Sd}~ww) a Run the Chain or language model synchronously. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. Returns: Union[List[dict], List[str], List[LLMResult], List[ChatResult]]: The outputs of the model or chain. r!r.Nr rr rr"r#z Error Type: z , Message: r%) rirr-rrhr/r&rr7rrr'rv) rrrrr)rsrCrro error_typer;r;r<_run_llm_or_chainsN    r1clientr project_nameproject_metadatadataset_versionOptional[Union[str, datetime]]1Tuple[MCF, TracerSession, Dataset, List[Example]]c Csbt||}|j|d}t|j|j|d} | std|ddd| D} | r,t| nd} | r4| nd} z'|p:i}t} | rHi|d| i}| |d<|j ||j|rWd |ini|d }Wn1t tt fy}z"d t |vrp|t }d |d |d|d}td|d|d}~ww|jd|j}td|d|d|d|jdd|||| fS)N)r) dataset_idas_ofzDataset z has no example rows.cSsg|]}|jr|jqSr;) modified_at)rFexr;r;r<rHsz%_prepare_eval_run..gitr5r)reference_dataset_id project_extrar zalready exists z+ run_on_dataset( ... project_name="z - z", # Update since z already exists ) z Test project z/ already exists. Please use a different name: z/compare?selectedSessions=z)View the evaluation results for project 'z' at: z View all tests for Dataset z at: T)flush)r read_datasetr list_examplesr'rmax isoformatrcreate_projectr,r+ruuiduuid4urlprint)r2rrr3r4rr5 wrapped_modeldatasetrr:max_modified_atinferred_versiongit_infoprojectrouid example_msgcomparison_urlr;r;r<_prepare_eval_runsl    rRc@s*eZdZUdZded<ded<ded<dS) _RowResultz5A dictionary of the results for a single example row.z Optional[List[EvaluationResult]]rVzOptional[float]rarrbNrr;r;r;r<rSs  rSF)totalc@seZdZUdZded<ded<ded<ded <d ed <d Zd ed<d>ddZd?ddZd@ddZdAd!d"Z dBdCd&d'Z e ( dDdEdrjcCsi}t|j|D]M\}}tt|t|ji}|j|dg|d|dd|t|j<t|t r?|j |t|jd<n ||t|jd<|j rU|j |t|jd<q|S)NrVrarb)rBrVrarbr`rCrD) ziprr rSrhrr'rrirvr`r)rQrXrYrUrrC row_resultr;r;r<_merge_test_outputss  z(_DatasetRunContainer._merge_test_outputsrunsDict[str, Run] List[dict]c s|j}|sgSfdd|jD}g}tjX}|D]L}z+|||j}t|tr-|}|t t||j |j j fi|d|j jdWqtyg}ztdt|d|WYd}~qd}~wwWd|S1sswY|S)Ncsg|] }t|jqSr;)rr'rFrr^r;r<rH1sz>_DatasetRunContainer._run_batch_evaluators..)rb project_idzError running batch evaluator z: )rWr concurrentfuturesThreadPoolExecutorrir!rjrlr submitr2create_feedbackrNr'r&rrcr() rQr^r runs_listaggregate_feedbackexecutor evaluatorrsror;rbr<_run_batch_evaluators-s>    z*_DatasetRunContainer._run_batch_evaluators,Tuple[Dict[str, _RowResult], Dict[str, Run]]c Csi}i}|jD]d}tt|dD]Z}t|tr3|j}|D]\\}}}|t|i d|iqqt|t rj|j } | rH| j rH| j | j nd} | rQt| jnd} |t|ji | | | d| |t|j<qqttttf||fS)Nr rV)rarbrun)rVr rrirlogged_eval_resultsrg setdefaultrrkr latest_runend_time start_time total_secondsr'rrrrS) rQrYall_runsccallback eval_results_rrrZrorarbr;r;r<_collect_metricsFs<   z%_DatasetRunContainer._collect_metrics-List[Union[dict, str, LLMResult, ChatResult]]r=cCsXtdt|\}}d}|jrtd||}|||}t|jj ||dS)Nz#Waiting for evaluators to complete.zRunning session evaluators.)r3rUaggregate_metrics) rrrr{rWrmr]r=rNr~)rQrXrYrvrjrUr;r;r<_collect_test_resultscs     z*_DatasetRunContainer._collect_test_resultsFverboseboolc Cs||}|r.z |}t|Wnty-}ztdt|WYd}~nd}~wwz|jj|j j t t jdW|Sty\}ztdt|WYd}~|Sd}~ww)Nz$Failed to print aggregate feedback: )rszFailed to close project: )r~rT_display_aggregate_resultsr&rrr(r2update_projectrNr'rnowrutc)rQrXrrU agg_feedbackror;r;r<finishus&   z_DatasetRunContainer.finishrrrrr3rr3rrrrrconcurrency_levelintr4r revision_idr5Optional[Union[datetime, str]]c  s|pt}| r| s i} | d| it|||| | d\} } }p%gjdp-iD]\}}d|d|q0djdi| rM| d<t|} t | ||| j pZt j t |d| |tt|fdd |D}|| |||r|jd Sdd S) Nr)r4rr5r<zgit:=r5rc sBg|]}ttj|jdtpg|jddgdqS))r3r2rrr)rr2rrmax_concurrency)r rrr )rrr~r'rrar2r progress_barrNr run_metadatarr;r<rHs*z0_DatasetRunContainer.prepare..)r2rNrIrrVrW)r4 random_namerkrRr rhrgrlrrrr'kvrr5ProgressBarCallbackrrW)clsr2rrr3r3rrrr4rr5rIrJrrYrZrVr;rr<preparesN  z_DatasetRunContainer.prepare)rXrrYrZr>rj)r^r_r>r`)r>rn)rXr|r>r=)F)rXrrrr>r=)NNNrNNN)r2rrrrrr3rr3rrrrrrrr4rrrr5rr>rU) r7r8r9r:rrWr]rmr{r~r classmethodrr;r;r;r<rU s,       rUrcCsDzddlm}|}|duodtt|vWSty!YdSw)Nr) get_ipythonzmqshellF)IPythonrrrrf)rresr;r;r<_is_jupyter_environments  raggregate_resultsr?cCsTtrddlm}m}||d||dS|jdddd}tdt|dS) Nr)HTMLdisplayz

Experiment Results:

cSs|dS)Nz.2fr;)xr;r;r<rrz,_display_aggregate_results..right) float_formatjustifyz Experiment Results:)rIPython.displayrr to_stringrH)rrrformatted_stringr;r;r<rs   raThe input_mapper argument is deprecated and will be removed in a future release. Please add a RunnableLambda to your chain to map inputs to the expected format instead. Example: def construct_chain(): my_chain = ... input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x} return input_mapper | my_chain run_on_dataset(..., llm_or_chain_factory=construct_chain) (See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r)r3r5rr3r4rrOptional[Client]rrrrrrxrc s| dd} | rtdtdd| durtd} | dd} | r)tdddd| r8tdd | d dd |p.rr)rrrr rhrrrUrr[rrVrget_executor_for_configrrrrr1rIr)r2rrr3r5rr3r4rrrxrrXrkr;rr<run_on_dataset9sh        ra1 Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to add to the project. Useful for storing information the test variant. (prompt version, model version, etc.) client: LangSmith client to use to access the dataset and to log feedback and run traces. verbose: Whether to print progress. tags: Tags to add to each run in the project. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see :func:`arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain_openai import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = smith_eval.RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", smith_eval.RunEvalConfig.Criteria("helpfulness"), smith_eval.RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = smith_eval.RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) zrun_on_dataset(zawait arun_on_dataset()r)rrrrr>r)rrr>r)rrr>rj)rr(rrr>ry)rr(rr.rrr>ry)rr(rrrrr>ry) rrrrr3rrr'r>r)rrrrr>r)rrrrr>r)rrrrr>r)rrrrrrrr'rrrrrrrrr>r") rrrrrrrrr>r)rrrrrr'rrrrrrr>r)rrrrrrr rrrr rr>r)rrrrr rrrrrr rr>r) rr(rrrrrrr>r )rrrrr rrrrrr rr>r)rrrrr rrrrrr rr>r.r)r2rrrrrr3rr4rrrr5r6r>r7)r>r)rr?r>ry)r2rrrrrr3rr5rrrr3rr4rrrrrrxrr>r)r: __future__rconcurrent.futuresrd dataclassesrrloggingrErrtypingrrrrr r r r r langchain_core._apir langchain_core.callbacks.managerrlangchain_core.language_modelsrlangchain_core.messagesrrlangchain_core.outputsrrlangchain_core.runnablesrrrrrrr!langchain_core.tracers.evaluationrr langchain_core.tracers.langchainrlangsmith.clientr langsmith.envrr langsmith.evaluationr!r"r#rlangsmith.run_helpersr$r%langsmith.schemasr&r'r(r)r*langsmith.utilsr+requestsr,typing_extensionsr-langchain.chains.baser.langchain.evaluation.loadingr/langchain.evaluation.schemar0r1r2langchain.smithr3rlangchain.smith.evaluationrr4r5rern getLoggerr7rrjrrr&r6r=rvrrrrrrrrrrrrrrrrr*r-r/r1rRrS dataclassrUrrrrr_RUN_ON_DATASET_DOCSTRINGreplacer;r;r;r<s ,                I =3 ( ! %      C GE%>C%= = C FMk