暾j,dZddlmZddlmZddlZddlZddlZ ddl m Z m Z m Z  ddlZddlZddlmZe rddlmZmZddlmZej2eZd Zd Zd ZGd d e ZddZ ddZ!ef ddZ"ddZ#ddZ$GddejJZ&GddejJZ'Gdde(Z)y#e$rdZYwxYw)zyImplements reading and writing to/from WebHDFS. The main entry point is the :func:`~smart_open.webhdfs.open` function. ) annotationsN) TYPE_CHECKINGAny TypedDictT) constants)ReadableBufferWriteableBuffer)TransportParamswebhdfs)zwebhdfs://host:port/path/filei c"eZdZUded<ded<y) _WebHDFSUristrschemeuriN)__name__ __module__ __qualname____annotations__\/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/smart_open/webhdfs.pyr r ,s K Hrr ct|dS)z>Return the WebHDFS URI as a dict with `scheme` and `uri` keys.)rr)SCHEME) uri_as_strs r parse_urir1s Z 00rcdtjjt|}t||fi|S)z=Open a WebHDFS URI using the given mode and transport params.) smart_openutils check_kwargsopen)rmodetransport_paramskwargss ropen_urir$6s/   * *41A BF T $V $$rc|jtr t|}|tjk(r t |}n2|tj k(rt||}nd|d}t||jdd|_ |S)aOpen a WebHDFS URI for reading or writing. Args: http_uri: webhdfs url converted to http REST url. mode: The mode for opening the object. Must be either "rb" or "wb". min_part_size: For writing only. Returns: A file-like object for reading from or writing to the WebHDFS file. Raises: NotImplementedError: If the requested mode is not supported. ) min_part_sizezwebhdfs support for mode z not implemented/) startswithr_convert_to_http_urir READ_BINARYBufferedInputBase WRITE_BINARYBufferedOutputBaseNotImplementedErrorsplitname)http_urir!r&fobjmsgs rr r >s 6"'1 y$$$ * '' '!(-H)$1AB!#&&s#B'DI Krctjj|}|jxsd}|jr|d|jz }|j }|j r6||rdnddztjj|j zz }tjjd|d|jz|dfS)zConvert webhdfs uri to http url and return it as text. Args: webhdfs_url: A URL starting with webhdfs://. Returns: The converted HTTP URL as a string. :&z user.name=httpz /webhdfs/v1) urllibparseurlsplithostnameportqueryusernamequote urlunsplitpath) webhdfs_url split_urinetlocr?s rr*r*^s %%k2I    %2F~~Ainn%&& OOE #B,69K9KIL^L^9___ << " "FFMINN4RTY[]#^ __rc,t|jS)zCConvert a parsed webhdfs URI to its HTTP REST URL (compat wrapper).)r*r) parsed_uris rconvert_to_http_urirIus   //rceZdZUdZded<dZded<ddZddZedd Z dd Z dd Z dd Z ddd Z dddZddZddZy)r,zGBuffered WebHDFS reader implementing the `io.BufferedIOBase` interface.rr1Nz bytes | None_bufc||_ddd}tj|j|d|_|jjt j k7rtj|jd|_ y)NOPENr)opoffsetT)paramsstreamr) _urirequestsget _response status_codehttplibOKWebHdfsException from_responserK)selfrpayloads r__init__zBufferedInputBase.__init__s] 1-!diiM >> % % 3"00@ @ rcVtjd|jsd|_yy)zFlush and close this stream.z close: calledN)loggerdebugclosedrKr[s rclosezBufferedInputBase.closes! _%{{DIrc|jduS)$Return True if the stream is closed.N)rKrbs rrazBufferedInputBase.closedsyyD  rcy)z+Return True if the stream can be read from.Trrbs rreadablezBufferedInputBase.readablercy)z:Return False; the WebHDFS reader does not support seeking.Frrbs rseekablezBufferedInputBase.seekablesrc"tj) Unsupported.ioUnsupportedOperationrbs rdetachzBufferedInputBase.detachs%%%rc|j}|J||dkr1d||jjjzc|_}|S|t |kr||d|d|c|_}|S|g} d}||krg|jjjt j }t |dk(rn%|t |z }|j|||krgdj|}||d|d|c|_}|S#t$rY.wxYw)zCRead up to `size` bytes (or all remaining bytes if `size` is None).Nrr) rKrUrawreadlenrnDEFAULT_BUFFER_SIZEappend StopIterationjoin)r[sizebufretvalbuffers total_readraw_datamergeds rrszBufferedInputBase.reads&ii <4!8 #S4>>+=+=+B+B+D%D DIvM #c(? #DE CJ DIvM% Jt#>>--2223I3IJx=A%c(m+ x(t#'""45M6%4= 6    s-A-C>> D  D c&|j|S)zThis is the same as read().)ry)rs)r[rys rread1zBufferedInputBase.read1syydy##rct|jd}|jt|}|sy||dt|t|S)zIRead up to ``len(b)`` bytes into `b` and return the number of bytes read.BrN) memoryviewcastrsrt)r[bmvdatas rreadintozBufferedInputBase.readintosG ]   $yyR!;SY4yrc|j}|Jd||jjjzc|_}|S)z1Read and return one line from the WebHDFS stream.r)rKrUrrreadline)r[rzr{s rrzBufferedInputBase.readlines@iit~~'9'9'B'B'D!D 6 r)rrreturnNonerrrboolrz io.RawIOBaseN)ry int | Nonerbytes)r()rr rint)rr)rrr__doc__rrKr]rcpropertyrargrjrprsrrrrrrr,r,zsUQ ID, !!&<$rr,cbeZdZUdZded<efd dZd dZddZddZ ddZ dd Z e d d Z y )r.a\Writes bytes to a WebHDFS file in multipart chunks. Args: uri: The HTTP WebHDFS REST URL to write to. min_part_size: The minimum part size for multipart uploads. For writing only. Raises: WebHdfsException: If the WebHDFS server returns an unexpected status code when creating the file. rr1c||_d|_||_ddd}tj|j|d}|j t jk(stj||jd}tj|ddd i }|j t jk(stj|g|_ d |_ d |_d |_y) NFCREATET)rN overwriterPallow_redirectslocationr6 content-typeapplication/octet-streamrheadersr)rR_closedr&rSputrVrWTEMPORARY_REDIRECTrYrZrCREATEDlinesparts chunk_bytes total_size)r[rr&r\ init_responseresponses rr]zBufferedOutputBase.__init__s  *!5 TYYwPUV ((G,F,FF"00? ?##J/<<"~Ga6bc##w6"00: :"$  rcy)z+Return True if the stream supports writing.Trrbs rwritablezBufferedOutputBase.writablerhrc0d}tj|)rlzdetach() not supportedrm)r[r4s rrpzBufferedOutputBase.detachs&%%c**rcjddi}tj|j|d}|jtj k(st j||jd}tj||ddi}|jtjk(st j|y) NrNAPPENDFrrrrr) rSpostrRrVrWrrYrZrrX)r[rr\rrrs r_uploadzBufferedOutputBase._upload s" diiQVW ((G,F,FF"00? ?##J/==4.Jd9ef##wzz1"00: :2rc|jr d}t|t|ts d}t ||j j ||xjt|z c_|xjt|z c_ |j|jk\rdj|j }tjd|jt||jdz |j|tj!d|j|xjdz c_gdc|_|_t|S) zMWrite the given bytes (binary string) into the WebHDFS file from constructor.zI/O operation on closed filezinput must be a binary stringrz+uploading part #%i, %i bytes (total %.3fGB)Azupload of part #%i finishedr)r ValueError isinstancer TypeErrorrrvrrtrr&rxr_inforrr`)r[rr4buffs rwritezBufferedOutputBase.writes <<0CS/ !!U#1CC.  ! CF" 3q6!   t11 188DJJ'D KK= D )+   LL  LL6 C JJ!OJ+-q (DJ(1v rcdj|j}|ritjd|jt ||j dz |j|tjd|jd|_ y)zCFlush any remaining buffered bytes to WebHDFS and close the stream.rz0uploading last part #%i, %i bytes (total %.3fGB)rz upload of last part #%i finishedTN) rxrr_rrrtrrr`r)r[rs rrczBufferedOutputBase.close0shxx #  KKB D )+   LL  LL;TZZ H rc|jS)re)rrbs rrazBufferedOutputBase.closed>s||rN)rrr&rrrrr)rrrr)rrrrr)rrrrr MIN_PART_SIZEr]rrprrrcrrarrrr.r.sC  I6C*+ ;: rr.c@eZdZdZddfd ZddZeddZxZS) rYzEException raised when WebHDFS returns an unexpected HTTP status code.cR||_||_t| t |yr)r4rVsuperr]repr)r[r4rV __class__s rr]zWebHdfsException.__init__Gs#& d$rch|jjd|jd|jdS)z6Return an unambiguous representation of the exception.z (status_code=z, msg=))rrrVr4rbs r__repr__zWebHdfsException.__repr__Ls5..))*-8H8H7IPTPXPX|[\]]rc>||j|jS)z=Build a `WebHdfsException` from a failed `requests.Response`.)r4rV)textrV)clsrs rrZzWebHdfsException.from_responsePsx}}(2F2FGGr)r6N)r4rrVrrr)rr)rzrequests.ResponserrY) rrrrr]r classmethodrZ __classcell__)rs@rrYrYDs'O% ^HHrrY)rrrr )rrr!rr"r r&BufferedInputBase | BufferedOutputBase)r2rr!rr&rrr)rDrrr)rHrrr)*r __future__r http.clientclientrWrnlogging urllib.parser:typingrrrrS ImportError MISSING_DEPSsmart_open.utilsrr _typeshedrr smart_open._typingr getLoggerrr_r URI_EXAMPLESrr rr$r r*rIBufferedIOBaser,r. ExceptionrYrrrrs # 00 92   8 $ 1   ) 1 % %%+:%+%4A-0+@`.0 \))\~h**hVHyHY LsCC  C