diff --git a/pytest/test_cross_types.py b/pytest/test_cross_types.py new file mode 100644 index 00000000..ceef22d8 --- /dev/null +++ b/pytest/test_cross_types.py @@ -0,0 +1,67 @@ +"""Python 2 Unicode literal representations on Python 3 hosts.""" + +import ast +import io +from struct import pack + +import pytest +from xdis.cross_types import UnicodeForPython3 +from xdis.magics import by_version, magic2int +from xdis.unmarshal import VersionIndependentUnmarshaller + +# Also verified by parsing and re-marshalling with stock CPython 2.7.18. +UNICODE_REPR_CASES = ( + ("", "u''"), + ("plain ASCII", "u'plain ASCII'"), + ("\u4e00\u6b21\u6700\u591a", r"u'\u4e00\u6b21\u6700\u591a'"), + ("abc\u4e2d\u6587", r"u'abc\u4e2d\u6587'"), + ("\u4e2d", r"u'\u4e2d'"), + ("\u00e9", r"u'\xe9'"), + ("\u0100", r"u'\u0100'"), + ("\U0001f600", r"u'\U0001f600'"), + ("\U0010ffff", r"u'\U0010ffff'"), + ("it's", 'u"it\'s"'), + ('say "hi"', """u'say "hi"'"""), + ("both ' and \"", "u'both \\' and \"'"), + ("back\\slash", r"u'back\\slash'"), + ("\\u4e2d", r"u'\\u4e2d'"), + ("line\nbreak", r"u'line\nbreak'"), + ("carriage\rreturn", r"u'carriage\rreturn'"), + ("nul\0tab\t", r"u'nul\x00tab\t'"), + ("\x01\x1f\x7f", r"u'\x01\x1f\x7f'"), + ("e\u0301", r"u'e\u0301'"), + ("\u2028\u2029", r"u'\u2028\u2029'"), + ("\u4e2d'\\\n", r'''u"\u4e2d'\\\n"'''), +) + + +@pytest.mark.parametrize("text, expected", UNICODE_REPR_CASES) +def test_python2_unicode_repr_roundtrip(text, expected): + raw = text.encode("utf-8") + value = UnicodeForPython3(raw) + literal = repr(value) + assert literal == expected + assert literal.isascii() + assert ast.literal_eval(literal) == text + assert str(value) == text + assert value.value == raw + + +@pytest.mark.parametrize("version", ("2.7", "3.3")) +@pytest.mark.parametrize( + "text", ("\u4e00\u6b21\u6700\u591a", "abc\u4e2d\u6587", "\u4e2d") +) +def test_marshaled_unicode_repr_roundtrip(version, text): + raw = text.encode("utf-8") + data = b"u" + pack(" int: return id(self.value) def __repr__(self) -> str: - r""" - Replacement repr() for Python3. - This ensures we get the "u" suffix on unicode types, - and also \u when the string is not ASCII representable - """ + """Return an ASCII-only Python 2 Unicode literal for UTF-8 text.""" try: utf8_value = self.value.decode("utf-8") # Do we need to handle utf-16 and utf-32? except UnicodeDecodeError: return f"""u'{str(self.value)[1:]}'""" - if is_ascii(utf8_value): - return f"""u'{utf8_value}'""" - - # Turn the unicode character into its Unicode code point, - # but strip of the leading "0x". - stripped_utf8 = utf8_value[len("0x") :] - unicode_codepoint = "".join( - (c if is_ascii(c) else hex(ord(c)) for c in stripped_utf8) - ) - return rf"""u'\u{unicode_codepoint}'""" + return "u" + ascii(utf8_value) def __str__(self) -> str: try: