o Jjg>@sUddlZddlmZddlmZmZmZmZmZm Z m Z m Z m Z m Z er1ddlmZmZmZmZhdZe eed<Gddde ZGd d d ZdS) N)platform) TYPE_CHECKINGAnyDictIterableListOptionalSetTuple TypedDictUnion)Browser CDPSessionPagesync_playwright> ::markerbrsvgbodyheadhtmlmetapathstyletitleiframescriptblack_listed_elementsc@sneZdZUdZeed<eed<eeed<eeed<eeed<e ed<eed<eed <eed <eed <d S) ElementInViewPortzIA typed dictionary containing information about elements in the viewport. node_indexbackend_node_id node_name node_value node_meta is_clickableorigin_xorigin_ycenter_xcenter_yN) __name__ __module__ __qualname____doc__str__annotations__intrrboolr1r1W/var/www/html/zoom/venv/lib/python3.10/site-packages/langchain/chains/natbot/crawler.pyr$s     rc@seZdZdZdddZdeddfddZd eddfd d Zd eee fddfd dZ d eee fdeddfddZ dddZ de efddZdS)Crawlera A crawler for web pages. **Security Note**: This is an implementation of a crawler that uses a browser via Playwright. This crawler can be used to load arbitrary webpages INCLUDING content from the local file system. Control access to who can submit crawling requests and what network access the crawler has. Make sure to scope permissions to the minimal permissions necessary for the application. See https://python.langchain.com/docs/security for more information. returnNcCshzddlm}Wn tytdw|jjdd|_|j|_|j ddd||dS) Nr)rz\Could not import playwright python package. Please install it with `pip install playwright`.F)headlessii8)widthheight) playwright.sync_apir ImportErrorstartchromiumlaunchbrowsernew_pagepageset_viewport_size)selfrr1r1r2__init__Es  zCrawler.__init__urlcCs:|jjd|vr |nd|d|jj|j|_i|_dS)Nz://zhttp://)rC)r?gotocontextnew_cdp_sessionclientpage_element_buffer)rArCr1r1r2 go_to_pageUs zCrawler.go_to_page directioncCs4|dkr |jddS|dkr|jddSdS)Nupz(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop - window.innerHeight;downz(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;)r?evaluate)rArJr1r1r2scrollZszCrawler.scrollidcCsTd}|j||jt|}|r$|d}|d}|jj||dStddS)Nz links = document.getElementsByTagName("a"); for (var i = 0; i < links.length; i++) { links[i].removeAttribute("target"); } r'r(zCould not find element)r?rMrHgetr/mouseclickprint)rArOjselementxyr1r1r2rRds  z Crawler.clicktextcCs|||jj|dS)N)rRr?keyboardtype)rArOrXr1r1r2rZws z Crawler.typecCs|jjddS)NEnter)r?rYpress)rAr1r1r2enter{sz Crawler.entercKs|j}|j}t}g}|d}tdkr|dkrd}|d}|d}|d}|d} ||} || } d} d} |d d d t| dt| d |j d gd d d}|d|dd }|d}|d}|d}|d}|d|dt |dd}|d}|d}|d}|d}|d}|d}d }i}g}ddi}ddi}d t t d!t t d"t fd#d$} dtttfd%tt d"tt t fffd&d' }!d(tt tt t tffd)t d*td t t d+td"tt t tff fd,d- tD]l\}"}#|"}$|#}%|d.|"|%|$\}&}'|d/|"|%|$\}(})z||"}WnYq|%tvr0q||\}*}+},}-|*|}*|+|}+|,|},|-|}-|*}.|+}/|*|,}0|+|-}1|.| kof|0|kof|/| kof|1|k}2|2skqg}3|!||"gd0}4|&pz|(}5|5sdn |&rt |'nt |)}6|5sdn|t |6g}7|%d1kr|5r|7r||"}8|8d2ks|8d3krq|7d4|8d5n:|%d6kr|4d4d7ks|%d/krd/}%|4d4d|4D]}9|5r|7r|7d8|9|4|9d9q|3|4|9qd}:||"d kr ||"}:|:d2kr qn&|%d6kr3|"|vr3|:dur3||"};||;}<|;d kr3||D]}?|?d;}@|?d }%|?d<}:|?d=}A|?d>}B|:r|:d?nd@}Cd@}D|@|vr||@D].}E|Ed4}F|Ed}G|Fd8kr|Br|EdA}H|B|HdB|GdCq|C|Gd?7}Cq|Brd?|B}Id?|I}D|Cd@kr|C}C| |%|A}J|Jd/ks|Dd@kr|JdDkr|Jd6kr|JdEkr|JdFkr|Cd@krqn|?||><|Cd@kr.|=dG|JdH|>|DdI|CdJ|JdI n|=dG|JdH|>|DdK|>d7}>qntdLt||=S)MNzwindow.devicePixelRatiodarwinzwindow.pageYOffsetzwindow.pageXOffsetzwindow.screen.widthzwindow.screen.heightrz[scrollbar {:0.2f}-{:0.2f}%])rVrWrXzDOMSnapshot.captureSnapshotT)computedStylesincludeDOMRectsincludePaintOrderstrings documentsnodes backendNodeId attributes nodeValue parentIndexnodeName isClickableindex inputValuevaluelayout nodeIndexboundsz-1FNr!has_click_handlerr4cSs8|dkrdS|dkr dS|dkrdS|dks|rdSdS)NalinkinputimgbuttonrXr1)r!rtr1r1r2 convert_names z#Crawler.crawl..convert_namekeyscsdi}tt|fdD]$\}}|dkrq |}|}||vr/|||<|||s/|Sq |S)Nr`r)zipiterremove)rhr{values key_index value_indexkeyro)rdr1r2find_attributess z&Crawler.crawl..find_attributes hash_treetagnode_id parent_idc sxt|}||vr|}|}|||||||\}} ||kr+d|f} n |r2d| f} nd} | |t|<| S)NTrs)r-lower) rrrr!r parent_id_str parent_namegrand_parent_idis_parent_desc_anchor anchor_idroadd_to_hash_tree node_namesparentrdr1r2rs      z'Crawler.crawl..add_to_hash_treerury)rZ placeholderz aria-labelraltz#text|u•rZ)rZrorwsubmit attribute)rZrro) rr r!r"r#r$r%r&r'r(rr"r$r# rz=""rvrxtextareazzParsing time: {:0.2f} seconds)r?rHtimerMrappendformatroundrGsendsetrr-r0rr/rrr enumeraterrmr setdefaultrPpopjoinstriprS)KrAr?rHr:page_state_as_textdevice_pixel_ratiowin_upper_boundwin_left_bound win_width win_heightwin_right_boundwin_lower_boundpercentage_progress_startpercentage_progress_endtreedocumentrfr rhr"r$ input_valueinput_value_indexinput_value_valuesrplayout_node_indexrrcursor child_nodeselements_in_view_portanchor_ancestrybutton_ancestryrzrrmnode_name_index node_parentr!is_ancestor_of_anchorris_ancestor_of_button button_idrVrWr6r7elem_left_boundelem_top_boundelem_right_boundelem_lower_boundpartially_is_in_viewport meta_dataelement_attributesancestor_exceptionancestor_node_key ancestor_noderXrelement_node_valuenode_input_text_index text_indexelements_of_interest id_counterrUrnode_is_clickablenode_meta_data inner_textrchild entry_type entry_value entry_key meta_stringconverted_node_namer1rr2crawl~s           #                                        z Crawler.crawl)r4N)r)r*r+r,rBr-rIrNr r/rRrZr]rrr1r1r1r2r33s   r3)rsysrtypingrrrrrrr r r r r8r rrrrr-r.rr3r1r1r1r2s  0