U do3@sddlZddlmZddlmZddlmZddlmZddl m Z ddl m Z mZ mZmZmZmZmZmZdd lmZmZdd lmZdd lmZmZmZGd d d ZGdddZeee fZ!ee!Z"GdddZ#dS)N)Counter)aliases)sha256)dumps)sub)AnyrDictIteratorListOptionalTupleUnion)NOT_PRINTABLE_PATTERNTOO_BIG_SEQUENCE) mess_ratio) iana_nameis_multi_byte_encoding unicode_rangec@s$eZdZdBeeeedeedddZe edddZ e edd d Z e ed d d Z e ed ddZe ed ddZe eed ddZed ddZed ddZdddddZe ed ddZe eed ddZe ed ddZe ed d d!Ze eed d"d#Ze ed d$d%Ze ed d&d'Ze ed d(d)Ze ed d*d+Ze ed d,d-Ze ed d.d/Z e edd d0d1Z!e ed d2d3Z"e eed d4d5Z#e eed d6d7Z$dd d8d9Z%dd d:d;Z&dCeed=d>d?Z'e ed d@dAZ(dS)D CharsetMatchNCoherenceMatches)payloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesdecoded_payloadcCsF||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ dS)N) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leavesZ_mean_coherence_ratio_output_payload_output_encoding_string)selfrrrrrrr)=/tmp/pip-unpacked-wheel-b4wixufl/charset_normalizer/models.py__init__s zCharsetMatch.__init__)otherreturncCs>t|ts&tdt|jt|j|j|jko<|j|jkS)Nz&__eq__ cannot be invoked on {} and {}.) isinstancer TypeErrorformatstr __class__encoding fingerprintr(r,r)r)r*__eq__1s zCharsetMatch.__eq__cCsvt|tstt|j|j}t|j|j}|dkrj|dkrj|dkr^|j|jkr^|j|jkS|j|jkS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gz?g{Gz?r)r.r ValueErrorabschaos coherencemulti_byte_usage)r(r,Zchaos_differenceZcoherence_differencer)r)r*__lt__:s   zCharsetMatch.__lt__r-cCsdtt|t|jS)N?)lenr1rawr(r)r)r*r;MszCharsetMatch.multi_byte_usagecCstdttt|dS)z Check once again chaos in decoded text, except this time, with full content. Use with caution, this can be very slow. Notice: Will be removed in 3.0 z=chaos_secondary_pass is deprecated and will be removed in 3.0r>)warningswarnDeprecationWarningrr1rAr)r)r*chaos_secondary_passQs z!CharsetMatch.chaos_secondary_passcCstdtdS)zy Coherence ratio on the first non-latin language detected if ANY. Notice: Will be removed in 3.0 z)r0r3r4rAr)r)r*__repr__~szCharsetMatch.__repr__cCs8t|tr||kr"td|jd|_|j|dS)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r.rr7r0r2r'r$appendr5r)r)r* add_submatchszCharsetMatch.add_submatchcCs|jSN)rrAr)r)r*r3szCharsetMatch.encodingcCsDg}tD]2\}}|j|kr*||q |j|kr ||q |S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr3rN)r(Z also_known_asupr)r)r*encoding_aliasess    zCharsetMatch.encoding_aliasescCs|jSrPr"rAr)r)r*bomszCharsetMatch.bomcCs|jSrPrUrAr)r)r*byte_order_markszCharsetMatch.byte_order_markcCsdd|jDS)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. cSsg|] }|dqS)rr)).0er)r)r* sz*CharsetMatch.languages..r!rAr)r)r*rszCharsetMatch.languagescCsp|jsbd|jkrdSddlm}m}t|jr8||jn||j}t|dksVd|krZdS|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiZEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedUnknown)r!could_be_from_charsetZcharset_normalizer.cdr]r^rr3r?)r(r]r^rr)r)r*languages  zCharsetMatch.languagecCs|jSrP)r rAr)r)r*r9szCharsetMatch.chaoscCs|js dS|jddS)Nrrrr[rAr)r)r*r:szCharsetMatch.coherencecCst|jdddSNd)ndigits)roundr9rAr)r)r* percent_chaosszCharsetMatch.percent_chaoscCst|jdddSrb)rfr:rAr)r)r*percent_coherenceszCharsetMatch.percent_coherencecCs|jS)z+ Original untouched bytes. )rrAr)r)r*r@szCharsetMatch.rawcCs|jSrP)r$rAr)r)r*submatchszCharsetMatch.submatchcCst|jdkSNr)r?r$rAr)r)r* has_submatchszCharsetMatch.has_submatchcCs@|jdk r|jSddt|D}ttdd|D|_|jS)NcSsg|] }t|qSr))r)rXcharr)r)r*rZsz*CharsetMatch.alphabets..cSsh|] }|r|qSr)r))rXrr)r)r* sz)CharsetMatch.alphabets..)r#r1sortedlist)r(Zdetected_rangesr)r)r* alphabetss zCharsetMatch.alphabetscCs|jgdd|jDS)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. cSsg|] }|jqSr))r3)rXmr)r)r*rZsz6CharsetMatch.could_be_from_charset..)rr$rAr)r)r*r`sz"CharsetMatch.could_be_from_charsetcCs|Sz> Kept for BC reasons. Will be removed in 3.0. r)rAr)r)r*firstszCharsetMatch.firstcCs|Srsr)rAr)r)r*bestszCharsetMatch.bestutf_8)r3r-cCs2|jdks|j|kr,||_t||d|_|jS)z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. Nreplace)r&r1encoder%)r(r3r)r)r*output szCharsetMatch.outputcCst|S)zw Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. )rry hexdigestrAr)r)r*r4szCharsetMatch.fingerprint)N)rv))__name__ __module__ __qualname__bytesr1floatboolr r+objectr6r<propertyr;rErF TypeCounterrJrLrMrOr3r rTrVrWrrar9r:rgrhr@rirkrqr`rtruryr4r)r)r)r*rsr         rc@seZdZdZdeeedddZeedddZ e e e fed d d Z e dd d ZedddZedd ddZeddddZeddddZdS)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. N)resultscCs|r t|ng|_dSrP)ro_results)r(rr)r)r*r+$szCharsetMatches.__init__r=ccs|jEdHdSrPrrAr)r)r*__iter__'szCharsetMatches.__iter__)itemr-cCsNt|tr|j|St|trFt|d}|jD]}||jkr.|Sq.tdS)z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. FN)r.intrr1rr`KeyError)r(rresultr)r)r* __getitem__*s       zCharsetMatches.__getitem__cCs t|jSrPr?rrAr)r)r*__len__8szCharsetMatches.__len__cCst|jdkSrjrrAr)r)r*__bool__;szCharsetMatches.__bool__cCs|t|tstdt|jt|jtkr`|j D],}|j |j kr2|j |j kr2| |dSq2|j |t|j |_ dS)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r.rr7r0r1r2r?r@rrr4r9rOrNro)r(rmatchr)r)r*rN>s    zCharsetMatches.appendrcCs|js dS|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. NrrrAr)r)r*ruRszCharsetMatches.bestcCs|S)zP Redundant method, call the method best(). Kept for BC reasons. )rurAr)r)r*rtZszCharsetMatches.first)N)r{r|r}__doc__r r rr+r rr rr1rrrrrNrurtr)r)r)r*rsrc @sjeZdZeeeeeeeeeeeeeeeed ddZe e ee fdddZ edddZ d S) CliDetectionResult pathr3rTalternative_encodingsrarqrr9r: unicode_path is_preferredc CsF||_| |_||_||_||_||_||_||_||_| |_ | |_ dSrP) rrr3rTrrarqrr9r:r) r(rr3rTrrarqrr9r:rrr)r)r*r+fszCliDetectionResult.__init__r=c Cs2|j|j|j|j|j|j|j|j|j|j |j d S)NrrrAr)r)r*__dict__szCliDetectionResult.__dict__cCst|jdddS)NT) ensure_asciiindent)rrrAr)r)r*to_jsonszCliDetectionResult.to_jsonN)r{r|r}r1r r rrr+rrrrrr)r)r)r*res r)$rB collectionsrZencodings.aliasesrhashlibrjsonrrertypingrrrr r r r r ZconstantrrZmdrutilsrrrrrr1rZCoherenceMatchrrr)r)r)r*s      (   C