Yi Kong | 8328301 | 2023-12-13 12:57:00 +0900 | [diff] [blame^] | 1 | """ Routines for manipulating RFC2047 encoded words. |
| 2 | |
| 3 | This is currently a package-private API, but will be considered for promotion |
| 4 | to a public API if there is demand. |
| 5 | |
| 6 | """ |
| 7 | |
| 8 | # An ecoded word looks like this: |
| 9 | # |
| 10 | # =?charset[*lang]?cte?encoded_string?= |
| 11 | # |
| 12 | # for more information about charset see the charset module. Here it is one |
| 13 | # of the preferred MIME charset names (hopefully; you never know when parsing). |
| 14 | # cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case). In |
| 15 | # theory other letters could be used for other encodings, but in practice this |
| 16 | # (almost?) never happens. There could be a public API for adding entries |
| 17 | # to the CTE tables, but YAGNI for now. 'q' is Quoted Printable, 'b' is |
| 18 | # Base64. The meaning of encoded_string should be obvious. 'lang' is optional |
| 19 | # as indicated by the brackets (they are not part of the syntax) but is almost |
| 20 | # never encountered in practice. |
| 21 | # |
| 22 | # The general interface for a CTE decoder is that it takes the encoded_string |
| 23 | # as its argument, and returns a tuple (cte_decoded_string, defects). The |
| 24 | # cte_decoded_string is the original binary that was encoded using the |
| 25 | # specified cte. 'defects' is a list of MessageDefect instances indicating any |
| 26 | # problems encountered during conversion. 'charset' and 'lang' are the |
| 27 | # corresponding strings extracted from the EW, case preserved. |
| 28 | # |
| 29 | # The general interface for a CTE encoder is that it takes a binary sequence |
| 30 | # as input and returns the cte_encoded_string, which is an ascii-only string. |
| 31 | # |
| 32 | # Each decoder must also supply a length function that takes the binary |
| 33 | # sequence as its argument and returns the length of the resulting encoded |
| 34 | # string. |
| 35 | # |
| 36 | # The main API functions for the module are decode, which calls the decoder |
| 37 | # referenced by the cte specifier, and encode, which adds the appropriate |
| 38 | # RFC 2047 "chrome" to the encoded string, and can optionally automatically |
| 39 | # select the shortest possible encoding. See their docstrings below for |
| 40 | # details. |
| 41 | |
| 42 | import re |
| 43 | import base64 |
| 44 | import binascii |
| 45 | import functools |
| 46 | from string import ascii_letters, digits |
| 47 | from email import errors |
| 48 | |
| 49 | __all__ = ['decode_q', |
| 50 | 'encode_q', |
| 51 | 'decode_b', |
| 52 | 'encode_b', |
| 53 | 'len_q', |
| 54 | 'len_b', |
| 55 | 'decode', |
| 56 | 'encode', |
| 57 | ] |
| 58 | |
| 59 | # |
| 60 | # Quoted Printable |
| 61 | # |
| 62 | |
| 63 | # regex based decoder. |
| 64 | _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, |
| 65 | lambda m: bytes.fromhex(m.group(1).decode())) |
| 66 | |
| 67 | def decode_q(encoded): |
| 68 | encoded = encoded.replace(b'_', b' ') |
| 69 | return _q_byte_subber(encoded), [] |
| 70 | |
| 71 | |
| 72 | # dict mapping bytes to their encoded form |
| 73 | class _QByteMap(dict): |
| 74 | |
| 75 | safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii') |
| 76 | |
| 77 | def __missing__(self, key): |
| 78 | if key in self.safe: |
| 79 | self[key] = chr(key) |
| 80 | else: |
| 81 | self[key] = "={:02X}".format(key) |
| 82 | return self[key] |
| 83 | |
| 84 | _q_byte_map = _QByteMap() |
| 85 | |
| 86 | # In headers spaces are mapped to '_'. |
| 87 | _q_byte_map[ord(' ')] = '_' |
| 88 | |
| 89 | def encode_q(bstring): |
| 90 | return ''.join(_q_byte_map[x] for x in bstring) |
| 91 | |
| 92 | def len_q(bstring): |
| 93 | return sum(len(_q_byte_map[x]) for x in bstring) |
| 94 | |
| 95 | |
| 96 | # |
| 97 | # Base64 |
| 98 | # |
| 99 | |
| 100 | def decode_b(encoded): |
| 101 | # First try encoding with validate=True, fixing the padding if needed. |
| 102 | # This will succeed only if encoded includes no invalid characters. |
| 103 | pad_err = len(encoded) % 4 |
| 104 | missing_padding = b'==='[:4-pad_err] if pad_err else b'' |
| 105 | try: |
| 106 | return ( |
| 107 | base64.b64decode(encoded + missing_padding, validate=True), |
| 108 | [errors.InvalidBase64PaddingDefect()] if pad_err else [], |
| 109 | ) |
| 110 | except binascii.Error: |
| 111 | # Since we had correct padding, this is likely an invalid char error. |
| 112 | # |
| 113 | # The non-alphabet characters are ignored as far as padding |
| 114 | # goes, but we don't know how many there are. So try without adding |
| 115 | # padding to see if it works. |
| 116 | try: |
| 117 | return ( |
| 118 | base64.b64decode(encoded, validate=False), |
| 119 | [errors.InvalidBase64CharactersDefect()], |
| 120 | ) |
| 121 | except binascii.Error: |
| 122 | # Add as much padding as could possibly be necessary (extra padding |
| 123 | # is ignored). |
| 124 | try: |
| 125 | return ( |
| 126 | base64.b64decode(encoded + b'==', validate=False), |
| 127 | [errors.InvalidBase64CharactersDefect(), |
| 128 | errors.InvalidBase64PaddingDefect()], |
| 129 | ) |
| 130 | except binascii.Error: |
| 131 | # This only happens when the encoded string's length is 1 more |
| 132 | # than a multiple of 4, which is invalid. |
| 133 | # |
| 134 | # bpo-27397: Just return the encoded string since there's no |
| 135 | # way to decode. |
| 136 | return encoded, [errors.InvalidBase64LengthDefect()] |
| 137 | |
| 138 | def encode_b(bstring): |
| 139 | return base64.b64encode(bstring).decode('ascii') |
| 140 | |
| 141 | def len_b(bstring): |
| 142 | groups_of_3, leftover = divmod(len(bstring), 3) |
| 143 | # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. |
| 144 | return groups_of_3 * 4 + (4 if leftover else 0) |
| 145 | |
| 146 | |
| 147 | _cte_decoders = { |
| 148 | 'q': decode_q, |
| 149 | 'b': decode_b, |
| 150 | } |
| 151 | |
| 152 | def decode(ew): |
| 153 | """Decode encoded word and return (string, charset, lang, defects) tuple. |
| 154 | |
| 155 | An RFC 2047/2243 encoded word has the form: |
| 156 | |
| 157 | =?charset*lang?cte?encoded_string?= |
| 158 | |
| 159 | where '*lang' may be omitted but the other parts may not be. |
| 160 | |
| 161 | This function expects exactly such a string (that is, it does not check the |
| 162 | syntax and may raise errors if the string is not well formed), and returns |
| 163 | the encoded_string decoded first from its Content Transfer Encoding and |
| 164 | then from the resulting bytes into unicode using the specified charset. If |
| 165 | the cte-decoded string does not successfully decode using the specified |
| 166 | character set, a defect is added to the defects list and the unknown octets |
| 167 | are replaced by the unicode 'unknown' character \\uFDFF. |
| 168 | |
| 169 | The specified charset and language are returned. The default for language, |
| 170 | which is rarely if ever encountered, is the empty string. |
| 171 | |
| 172 | """ |
| 173 | _, charset, cte, cte_string, _ = ew.split('?') |
| 174 | charset, _, lang = charset.partition('*') |
| 175 | cte = cte.lower() |
| 176 | # Recover the original bytes and do CTE decoding. |
| 177 | bstring = cte_string.encode('ascii', 'surrogateescape') |
| 178 | bstring, defects = _cte_decoders[cte](bstring) |
| 179 | # Turn the CTE decoded bytes into unicode. |
| 180 | try: |
| 181 | string = bstring.decode(charset) |
| 182 | except UnicodeError: |
| 183 | defects.append(errors.UndecodableBytesDefect("Encoded word " |
| 184 | "contains bytes not decodable using {} charset".format(charset))) |
| 185 | string = bstring.decode(charset, 'surrogateescape') |
| 186 | except LookupError: |
| 187 | string = bstring.decode('ascii', 'surrogateescape') |
| 188 | if charset.lower() != 'unknown-8bit': |
| 189 | defects.append(errors.CharsetError("Unknown charset {} " |
| 190 | "in encoded word; decoded as unknown bytes".format(charset))) |
| 191 | return string, charset, lang, defects |
| 192 | |
| 193 | |
| 194 | _cte_encoders = { |
| 195 | 'q': encode_q, |
| 196 | 'b': encode_b, |
| 197 | } |
| 198 | |
| 199 | _cte_encode_length = { |
| 200 | 'q': len_q, |
| 201 | 'b': len_b, |
| 202 | } |
| 203 | |
| 204 | def encode(string, charset='utf-8', encoding=None, lang=''): |
| 205 | """Encode string using the CTE encoding that produces the shorter result. |
| 206 | |
| 207 | Produces an RFC 2047/2243 encoded word of the form: |
| 208 | |
| 209 | =?charset*lang?cte?encoded_string?= |
| 210 | |
| 211 | where '*lang' is omitted unless the 'lang' parameter is given a value. |
| 212 | Optional argument charset (defaults to utf-8) specifies the charset to use |
| 213 | to encode the string to binary before CTE encoding it. Optional argument |
| 214 | 'encoding' is the cte specifier for the encoding that should be used ('q' |
| 215 | or 'b'); if it is None (the default) the encoding which produces the |
| 216 | shortest encoded sequence is used, except that 'q' is preferred if it is up |
| 217 | to five characters longer. Optional argument 'lang' (default '') gives the |
| 218 | RFC 2243 language string to specify in the encoded word. |
| 219 | |
| 220 | """ |
| 221 | if charset == 'unknown-8bit': |
| 222 | bstring = string.encode('ascii', 'surrogateescape') |
| 223 | else: |
| 224 | bstring = string.encode(charset) |
| 225 | if encoding is None: |
| 226 | qlen = _cte_encode_length['q'](bstring) |
| 227 | blen = _cte_encode_length['b'](bstring) |
| 228 | # Bias toward q. 5 is arbitrary. |
| 229 | encoding = 'q' if qlen - blen < 5 else 'b' |
| 230 | encoded = _cte_encoders[encoding](bstring) |
| 231 | if lang: |
| 232 | lang = '*' + lang |
| 233 | return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded) |