|
| 1 | +# This test covers backwards compatibility with previous versions of Python |
| 2 | +# by bouncing pickled objects through Python versions by running xpickle_worker.py. |
| 3 | +import io |
| 4 | +import os |
| 5 | +import pickle |
| 6 | +import struct |
| 7 | +import subprocess |
| 8 | +import sys |
| 9 | +import unittest |
| 10 | + |
| 11 | + |
| 12 | +from test import support |
| 13 | +from test import pickletester |
| 14 | + |
| 15 | +try: |
| 16 | + import _pickle |
| 17 | + has_c_implementation = True |
| 18 | +except ModuleNotFoundError: |
| 19 | + has_c_implementation = False |
| 20 | + |
| 21 | +support.requires('xpickle') |
| 22 | + |
| 23 | +is_windows = sys.platform.startswith('win') |
| 24 | + |
| 25 | +# Map python version to a tuple containing the name of a corresponding valid |
| 26 | +# Python binary to execute and its arguments. |
| 27 | +py_executable_map = {} |
| 28 | + |
| 29 | +protocols_map = { |
| 30 | + 3: (3, 0), |
| 31 | + 4: (3, 4), |
| 32 | + 5: (3, 8), |
| 33 | +} |
| 34 | + |
| 35 | +def highest_proto_for_py_version(py_version): |
| 36 | + """Finds the highest supported pickle protocol for a given Python version. |
| 37 | + Args: |
| 38 | + py_version: a 2-tuple of the major, minor version. Eg. Python 3.7 would |
| 39 | + be (3, 7) |
| 40 | + Returns: |
| 41 | + int for the highest supported pickle protocol |
| 42 | + """ |
| 43 | + proto = 2 |
| 44 | + for p, v in protocols_map.items(): |
| 45 | + if py_version < v: |
| 46 | + break |
| 47 | + proto = p |
| 48 | + return proto |
| 49 | + |
| 50 | +def have_python_version(py_version): |
| 51 | + """Check whether a Python binary exists for the given py_version and has |
| 52 | + support. This respects your PATH. |
| 53 | + For Windows, it will first try to use the py launcher specified in PEP 397. |
| 54 | + Otherwise (and for all other platforms), it will attempt to check for |
| 55 | + python<py_version[0]>.<py_version[1]>. |
| 56 | +
|
| 57 | + Eg. given a *py_version* of (3, 7), the function will attempt to try |
| 58 | + 'py -3.7' (for Windows) first, then 'python3.7', and return |
| 59 | + ['py', '-3.7'] (on Windows) or ['python3.7'] on other platforms. |
| 60 | +
|
| 61 | + Args: |
| 62 | + py_version: a 2-tuple of the major, minor version. Eg. python 3.7 would |
| 63 | + be (3, 7) |
| 64 | + Returns: |
| 65 | + List/Tuple containing the Python binary name and its required arguments, |
| 66 | + or None if no valid binary names found. |
| 67 | + """ |
| 68 | + python_str = ".".join(map(str, py_version)) |
| 69 | + targets = [('py', f'-{python_str}'), (f'python{python_str}',)] |
| 70 | + if py_version not in py_executable_map: |
| 71 | + for target in targets[0 if is_windows else 1:]: |
| 72 | + try: |
| 73 | + worker = subprocess.Popen([*target, '-c', 'pass'], |
| 74 | + stdout=subprocess.DEVNULL, |
| 75 | + stderr=subprocess.DEVNULL, |
| 76 | + shell=is_windows) |
| 77 | + worker.communicate() |
| 78 | + if worker.returncode == 0: |
| 79 | + py_executable_map[py_version] = target |
| 80 | + break |
| 81 | + except FileNotFoundError: |
| 82 | + pass |
| 83 | + |
| 84 | + return py_executable_map.get(py_version, None) |
| 85 | + |
| 86 | + |
| 87 | +def read_exact(f, n): |
| 88 | + buf = b'' |
| 89 | + while len(buf) < n: |
| 90 | + chunk = f.read(n - len(buf)) |
| 91 | + if not chunk: |
| 92 | + raise EOFError |
| 93 | + buf += chunk |
| 94 | + return buf |
| 95 | + |
| 96 | + |
| 97 | +class AbstractCompatTests(pickletester.AbstractPickleTests): |
| 98 | + py_version = None |
| 99 | + worker = None |
| 100 | + |
| 101 | + @classmethod |
| 102 | + def setUpClass(cls): |
| 103 | + assert cls.py_version is not None, 'Needs a python version tuple' |
| 104 | + if not have_python_version(cls.py_version): |
| 105 | + py_version_str = ".".join(map(str, cls.py_version)) |
| 106 | + raise unittest.SkipTest(f'Python {py_version_str} not available') |
| 107 | + cls.addClassCleanup(cls.finish_worker) |
| 108 | + # Override the default pickle protocol to match what xpickle worker |
| 109 | + # will be running. |
| 110 | + highest_protocol = highest_proto_for_py_version(cls.py_version) |
| 111 | + cls.enterClassContext(support.swap_attr(pickletester, 'protocols', |
| 112 | + range(highest_protocol + 1))) |
| 113 | + cls.enterClassContext(support.swap_attr(pickle, 'HIGHEST_PROTOCOL', |
| 114 | + highest_protocol)) |
| 115 | + |
| 116 | + @classmethod |
| 117 | + def start_worker(cls, python): |
| 118 | + target = os.path.join(os.path.dirname(__file__), 'xpickle_worker.py') |
| 119 | + worker = subprocess.Popen([*python, target], |
| 120 | + stdin=subprocess.PIPE, |
| 121 | + stdout=subprocess.PIPE, |
| 122 | + stderr=subprocess.PIPE, |
| 123 | + # For windows bpo-17023. |
| 124 | + shell=is_windows) |
| 125 | + cls.worker = worker |
| 126 | + return worker |
| 127 | + |
| 128 | + @classmethod |
| 129 | + def finish_worker(cls): |
| 130 | + worker = cls.worker |
| 131 | + if worker is None: |
| 132 | + return |
| 133 | + cls.worker = None |
| 134 | + worker.stdin.close() |
| 135 | + worker.stdout.close() |
| 136 | + worker.stderr.close() |
| 137 | + worker.terminate() |
| 138 | + worker.wait() |
| 139 | + |
| 140 | + @classmethod |
| 141 | + def send_to_worker(cls, python, data): |
| 142 | + """Bounce a pickled object through another version of Python. |
| 143 | + This will send data to a child process where it will |
| 144 | + be unpickled, then repickled and sent back to the parent process. |
| 145 | + Args: |
| 146 | + python: list containing the python binary to start and its arguments |
| 147 | + data: bytes object to send to the child process |
| 148 | + Returns: |
| 149 | + The pickled data received from the child process. |
| 150 | + """ |
| 151 | + worker = cls.worker |
| 152 | + if worker is None: |
| 153 | + worker = cls.start_worker(python) |
| 154 | + |
| 155 | + try: |
| 156 | + worker.stdin.write(struct.pack('!i', len(data)) + data) |
| 157 | + worker.stdin.flush() |
| 158 | + |
| 159 | + size, = struct.unpack('!i', read_exact(worker.stdout, 4)) |
| 160 | + if size > 0: |
| 161 | + return read_exact(worker.stdout, size) |
| 162 | + # if the worker fails, it will write the exception to stdout |
| 163 | + if size < 0: |
| 164 | + stdout = read_exact(worker.stdout, -size) |
| 165 | + try: |
| 166 | + exception = pickle.loads(stdout) |
| 167 | + except (pickle.UnpicklingError, EOFError): |
| 168 | + pass |
| 169 | + else: |
| 170 | + if isinstance(exception, Exception): |
| 171 | + # To allow for tests which test for errors. |
| 172 | + raise exception |
| 173 | + _, stderr = worker.communicate() |
| 174 | + raise RuntimeError(stderr) |
| 175 | + except: |
| 176 | + cls.finish_worker() |
| 177 | + raise |
| 178 | + |
| 179 | + def dumps(self, arg, proto=0, **kwargs): |
| 180 | + # Skip tests that require buffer_callback arguments since |
| 181 | + # there isn't a reliable way to marshal/pickle the callback and ensure |
| 182 | + # it works in a different Python version. |
| 183 | + if 'buffer_callback' in kwargs: |
| 184 | + self.skipTest('Test does not support "buffer_callback" argument.') |
| 185 | + f = io.BytesIO() |
| 186 | + p = self.pickler(f, proto, **kwargs) |
| 187 | + p.dump(arg) |
| 188 | + data = struct.pack('!i', proto) + f.getvalue() |
| 189 | + python = py_executable_map[self.py_version] |
| 190 | + return self.send_to_worker(python, data) |
| 191 | + |
| 192 | + def loads(self, buf, **kwds): |
| 193 | + f = io.BytesIO(buf) |
| 194 | + u = self.unpickler(f, **kwds) |
| 195 | + return u.load() |
| 196 | + |
| 197 | + # A scaled-down version of test_bytes from pickletester, to reduce |
| 198 | + # the number of calls to self.dumps() and hence reduce the number of |
| 199 | + # child python processes forked. This allows the test to complete |
| 200 | + # much faster (the one from pickletester takes 3-4 minutes when running |
| 201 | + # under text_xpickle). |
| 202 | + def test_bytes(self): |
| 203 | + if self.py_version < (3, 0): |
| 204 | + self.skipTest('not supported in Python < 3.0') |
| 205 | + for proto in pickletester.protocols: |
| 206 | + for s in b'', b'xyz', b'xyz'*100: |
| 207 | + p = self.dumps(s, proto) |
| 208 | + self.assert_is_copy(s, self.loads(p)) |
| 209 | + s = bytes(range(256)) |
| 210 | + p = self.dumps(s, proto) |
| 211 | + self.assert_is_copy(s, self.loads(p)) |
| 212 | + s = bytes([i for i in range(256) for _ in range(2)]) |
| 213 | + p = self.dumps(s, proto) |
| 214 | + self.assert_is_copy(s, self.loads(p)) |
| 215 | + |
| 216 | + # These tests are disabled because they require some special setup |
| 217 | + # on the worker that's hard to keep in sync. |
| 218 | + test_global_ext1 = None |
| 219 | + test_global_ext2 = None |
| 220 | + test_global_ext4 = None |
| 221 | + |
| 222 | + # These tests fail because they require classes from pickletester |
| 223 | + # which cannot be properly imported by the xpickle worker. |
| 224 | + test_recursive_nested_names = None |
| 225 | + test_recursive_nested_names2 = None |
| 226 | + |
| 227 | + # Attribute lookup problems are expected, disable the test |
| 228 | + test_dynamic_class = None |
| 229 | + test_evil_class_mutating_dict = None |
| 230 | + |
| 231 | + # Expected exception is raised during unpickling in a subprocess. |
| 232 | + test_pickle_setstate_None = None |
| 233 | + |
| 234 | + # Other Python version may not have NumPy. |
| 235 | + test_buffers_numpy = None |
| 236 | + |
| 237 | + # Skip tests that require buffer_callback arguments since |
| 238 | + # there isn't a reliable way to marshal/pickle the callback and ensure |
| 239 | + # it works in a different Python version. |
| 240 | + test_in_band_buffers = None |
| 241 | + test_buffers_error = None |
| 242 | + test_oob_buffers = None |
| 243 | + test_oob_buffers_writable_to_readonly = None |
| 244 | + |
| 245 | +class PyPicklePythonCompat(AbstractCompatTests): |
| 246 | + pickler = pickle._Pickler |
| 247 | + unpickler = pickle._Unpickler |
| 248 | + |
| 249 | +if has_c_implementation: |
| 250 | + class CPicklePythonCompat(AbstractCompatTests): |
| 251 | + pickler = _pickle.Pickler |
| 252 | + unpickler = _pickle.Unpickler |
| 253 | + |
| 254 | + |
| 255 | +def make_test(py_version, base): |
| 256 | + class_dict = {'py_version': py_version} |
| 257 | + name = base.__name__.replace('Python', 'Python%d%d' % py_version) |
| 258 | + return type(name, (base, unittest.TestCase), class_dict) |
| 259 | + |
| 260 | +def load_tests(loader, tests, pattern): |
| 261 | + def add_tests(py_version): |
| 262 | + test_class = make_test(py_version, PyPicklePythonCompat) |
| 263 | + tests.addTest(loader.loadTestsFromTestCase(test_class)) |
| 264 | + if has_c_implementation: |
| 265 | + test_class = make_test(py_version, CPicklePythonCompat) |
| 266 | + tests.addTest(loader.loadTestsFromTestCase(test_class)) |
| 267 | + |
| 268 | + value = support.get_resource_value('xpickle') |
| 269 | + if value is None: |
| 270 | + major = sys.version_info.major |
| 271 | + assert major == 3 |
| 272 | + add_tests((2, 7)) |
| 273 | + for minor in range(2, sys.version_info.minor): |
| 274 | + add_tests((major, minor)) |
| 275 | + else: |
| 276 | + add_tests(tuple(map(int, value.split('.')))) |
| 277 | + return tests |
| 278 | + |
| 279 | + |
| 280 | +if __name__ == '__main__': |
| 281 | + unittest.main() |