|
2 | 2 | To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been |
3 | 3 | created. Clearly there is huge need to use conforming syntax. |
4 | 4 | """ |
5 | | -import binascii |
6 | 5 | import errno |
7 | 6 | import sys |
8 | 7 | import os |
|
17 | 16 | pass |
18 | 17 |
|
19 | 18 | is_py3 = sys.version_info[0] >= 3 |
20 | | -is_py33 = is_py3 and sys.version_info[1] >= 3 |
21 | | -is_py34 = is_py3 and sys.version_info[1] >= 4 |
22 | 19 | is_py35 = is_py3 and sys.version_info[1] >= 5 |
23 | 20 | py_version = int(str(sys.version_info[0]) + str(sys.version_info[1])) |
24 | 21 |
|
@@ -35,24 +32,36 @@ def close(self): |
35 | 32 | del self.loader |
36 | 33 |
|
37 | 34 |
|
38 | | -def find_module_py34(string, path=None, full_name=None): |
| 35 | +def find_module_py34(string, path=None, full_name=None, is_global_search=True): |
39 | 36 | spec = None |
40 | 37 | loader = None |
41 | 38 |
|
42 | | - spec = importlib.machinery.PathFinder.find_spec(string, path) |
43 | | - if spec is not None: |
44 | | - # We try to disambiguate implicit namespace pkgs with non implicit namespace pkgs |
45 | | - if not spec.has_location: |
46 | | - full_name = string if not path else full_name |
47 | | - implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path) |
48 | | - return None, implicit_ns_info, False |
| 39 | + for finder in sys.meta_path: |
| 40 | + if is_global_search and finder != importlib.machinery.PathFinder: |
| 41 | + p = None |
| 42 | + else: |
| 43 | + p = path |
| 44 | + try: |
| 45 | + find_spec = finder.find_spec |
| 46 | + except AttributeError: |
| 47 | + # These are old-school clases that still have a different API, just |
| 48 | + # ignore those. |
| 49 | + continue |
| 50 | + |
| 51 | + spec = find_spec(string, p) |
| 52 | + if spec is not None: |
| 53 | + loader = spec.loader |
| 54 | + if loader is None and not spec.has_location: |
| 55 | + # This is a namespace package. |
| 56 | + full_name = string if not path else full_name |
| 57 | + implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path) |
| 58 | + return None, implicit_ns_info, False |
| 59 | + break |
49 | 60 |
|
50 | | - # we have found the tail end of the dotted path |
51 | | - loader = spec.loader |
52 | 61 | return find_module_py33(string, path, loader) |
53 | 62 |
|
54 | 63 |
|
55 | | -def find_module_py33(string, path=None, loader=None, full_name=None): |
| 64 | +def find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True): |
56 | 65 | loader = loader or importlib.machinery.PathFinder.find_module(string, path) |
57 | 66 |
|
58 | 67 | if loader is None and path is None: # Fallback to find builtins |
@@ -105,7 +114,7 @@ def find_module_py33(string, path=None, loader=None, full_name=None): |
105 | 114 | return module_file, module_path, is_package |
106 | 115 |
|
107 | 116 |
|
108 | | -def find_module_pre_py33(string, path=None, full_name=None): |
| 117 | +def find_module_pre_py34(string, path=None, full_name=None, is_global_search=True): |
109 | 118 | # This import is here, because in other places it will raise a |
110 | 119 | # DeprecationWarning. |
111 | 120 | import imp |
@@ -140,8 +149,7 @@ def find_module_pre_py33(string, path=None, full_name=None): |
140 | 149 | raise ImportError("No module named {}".format(string)) |
141 | 150 |
|
142 | 151 |
|
143 | | -find_module = find_module_py33 if is_py33 else find_module_pre_py33 |
144 | | -find_module = find_module_py34 if is_py34 else find_module |
| 152 | +find_module = find_module_py34 if is_py3 else find_module_pre_py34 |
145 | 153 | find_module.__doc__ = """ |
146 | 154 | Provides information about a module. |
147 | 155 |
|
@@ -208,6 +216,7 @@ def _iter_modules(paths, prefix=''): |
208 | 216 | yield importer, prefix + modname, ispkg |
209 | 217 | # END COPY |
210 | 218 |
|
| 219 | + |
211 | 220 | iter_modules = _iter_modules if py_version >= 34 else pkgutil.iter_modules |
212 | 221 |
|
213 | 222 |
|
@@ -253,6 +262,7 @@ def reraise(exception, traceback): |
253 | 262 |
|
254 | 263 | """ |
255 | 264 |
|
| 265 | + |
256 | 266 | class Python3Method(object): |
257 | 267 | def __init__(self, func): |
258 | 268 | self.func = func |
@@ -313,10 +323,10 @@ def force_unicode(obj): |
313 | 323 | try: |
314 | 324 | import builtins # module name in python 3 |
315 | 325 | except ImportError: |
316 | | - import __builtin__ as builtins |
| 326 | + import __builtin__ as builtins # noqa: F401 |
317 | 327 |
|
318 | 328 |
|
319 | | -import ast |
| 329 | +import ast # noqa: F401 |
320 | 330 |
|
321 | 331 |
|
322 | 332 | def literal_eval(string): |
@@ -326,7 +336,7 @@ def literal_eval(string): |
326 | 336 | try: |
327 | 337 | from itertools import zip_longest |
328 | 338 | except ImportError: |
329 | | - from itertools import izip_longest as zip_longest # Python 2 |
| 339 | + from itertools import izip_longest as zip_longest # Python 2 # noqa: F401 |
330 | 340 |
|
331 | 341 | try: |
332 | 342 | FileNotFoundError = FileNotFoundError |
@@ -356,6 +366,7 @@ def print_to_stderr(*args): |
356 | 366 | eval("print(*args, file=sys.stderr)") |
357 | 367 | else: |
358 | 368 | print >> sys.stderr, args |
| 369 | + sys.stderr.flush() |
359 | 370 |
|
360 | 371 |
|
361 | 372 | def utf8_repr(func): |
@@ -379,10 +390,14 @@ def wrapper(self): |
379 | 390 | if is_py3: |
380 | 391 | import queue |
381 | 392 | else: |
382 | | - import Queue as queue |
383 | | - |
| 393 | + import Queue as queue # noqa: F401 |
384 | 394 |
|
385 | | -import pickle |
| 395 | +try: |
| 396 | + # Attempt to load the C implementation of pickle on Python 2 as it is way |
| 397 | + # faster. |
| 398 | + import cPickle as pickle |
| 399 | +except ImportError: |
| 400 | + import pickle |
386 | 401 | if sys.version_info[:2] == (3, 3): |
387 | 402 | """ |
388 | 403 | Monkeypatch the unpickler in Python 3.3. This is needed, because the |
@@ -443,53 +458,45 @@ def loads(s, fix_imports=True, encoding="ASCII", errors="strict"): |
443 | 458 | pickle.loads = loads |
444 | 459 |
|
445 | 460 |
|
446 | | -_PICKLE_PROTOCOL = 2 |
447 | | -is_windows = sys.platform == 'win32' |
448 | | - |
449 | | -# The Windows shell on Python 2 consumes all control characters (below 32) and expand on |
450 | | -# all Python versions \n to \r\n. |
451 | | -# pickle starting from protocol version 1 uses binary data, which could not be escaped by |
452 | | -# any normal unicode encoder. Therefore, the only bytes encoder which doesn't produce |
453 | | -# control characters is binascii.hexlify. |
454 | | - |
455 | | - |
456 | 461 | def pickle_load(file): |
457 | | - if is_windows: |
458 | | - try: |
459 | | - data = file.readline() |
460 | | - data = binascii.unhexlify(data.strip()) |
461 | | - if is_py3: |
462 | | - return pickle.loads(data, encoding='bytes') |
463 | | - else: |
464 | | - return pickle.loads(data) |
465 | | - # Python on Windows don't throw EOF errors for pipes. So reraise them with |
466 | | - # the correct type, which is cought upwards. |
467 | | - except OSError: |
468 | | - raise EOFError() |
469 | | - else: |
| 462 | + try: |
470 | 463 | if is_py3: |
471 | 464 | return pickle.load(file, encoding='bytes') |
472 | | - else: |
473 | | - return pickle.load(file) |
| 465 | + return pickle.load(file) |
| 466 | + # Python on Windows don't throw EOF errors for pipes. So reraise them with |
| 467 | + # the correct type, which is caught upwards. |
| 468 | + except OSError: |
| 469 | + if sys.platform == 'win32': |
| 470 | + raise EOFError() |
| 471 | + raise |
474 | 472 |
|
475 | 473 |
|
476 | | -def pickle_dump(data, file): |
477 | | - if is_windows: |
478 | | - try: |
479 | | - data = pickle.dumps(data, protocol=_PICKLE_PROTOCOL) |
480 | | - data = binascii.hexlify(data) |
481 | | - file.write(data) |
482 | | - file.write(b'\n') |
483 | | - # On Python 3.3 flush throws sometimes an error even if the two file writes |
484 | | - # should done it already before. This could be also computer / speed depending. |
485 | | - file.flush() |
486 | | - # Python on Windows don't throw EPIPE errors for pipes. So reraise them with |
487 | | - # the correct type and error number. |
488 | | - except OSError: |
489 | | - raise IOError(errno.EPIPE, "Broken pipe") |
490 | | - else: |
491 | | - pickle.dump(data, file, protocol=_PICKLE_PROTOCOL) |
| 474 | +def pickle_dump(data, file, protocol): |
| 475 | + try: |
| 476 | + pickle.dump(data, file, protocol) |
| 477 | + # On Python 3.3 flush throws sometimes an error even though the writing |
| 478 | + # operation should be completed. |
492 | 479 | file.flush() |
| 480 | + # Python on Windows don't throw EPIPE errors for pipes. So reraise them with |
| 481 | + # the correct type and error number. |
| 482 | + except OSError: |
| 483 | + if sys.platform == 'win32': |
| 484 | + raise IOError(errno.EPIPE, "Broken pipe") |
| 485 | + raise |
| 486 | + |
| 487 | + |
| 488 | +# Determine the highest protocol version compatible for a given list of Python |
| 489 | +# versions. |
| 490 | +def highest_pickle_protocol(python_versions): |
| 491 | + protocol = 4 |
| 492 | + for version in python_versions: |
| 493 | + if version[0] == 2: |
| 494 | + # The minimum protocol version for the versions of Python that we |
| 495 | + # support (2.7 and 3.3+) is 2. |
| 496 | + return 2 |
| 497 | + if version[1] < 4: |
| 498 | + protocol = 3 |
| 499 | + return protocol |
493 | 500 |
|
494 | 501 |
|
495 | 502 | try: |
@@ -512,4 +519,71 @@ def __init__(self, *args, **kwargs): |
512 | 519 | except AttributeError: |
513 | 520 | CREATE_NO_WINDOW = 0x08000000 |
514 | 521 | kwargs['creationflags'] = CREATE_NO_WINDOW |
| 522 | + # The child process doesn't need file descriptors except 0, 1, 2. |
| 523 | + # This is unix only. |
| 524 | + kwargs['close_fds'] = 'posix' in sys.builtin_module_names |
515 | 525 | super(GeneralizedPopen, self).__init__(*args, **kwargs) |
| 526 | + |
| 527 | + |
| 528 | +# shutil.which is not available on Python 2.7. |
| 529 | +def which(cmd, mode=os.F_OK | os.X_OK, path=None): |
| 530 | + """Given a command, mode, and a PATH string, return the path which |
| 531 | + conforms to the given mode on the PATH, or None if there is no such |
| 532 | + file. |
| 533 | +
|
| 534 | + `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result |
| 535 | + of os.environ.get("PATH"), or can be overridden with a custom search |
| 536 | + path. |
| 537 | +
|
| 538 | + """ |
| 539 | + # Check that a given file can be accessed with the correct mode. |
| 540 | + # Additionally check that `file` is not a directory, as on Windows |
| 541 | + # directories pass the os.access check. |
| 542 | + def _access_check(fn, mode): |
| 543 | + return (os.path.exists(fn) and os.access(fn, mode) |
| 544 | + and not os.path.isdir(fn)) |
| 545 | + |
| 546 | + # If we're given a path with a directory part, look it up directly rather |
| 547 | + # than referring to PATH directories. This includes checking relative to the |
| 548 | + # current directory, e.g. ./script |
| 549 | + if os.path.dirname(cmd): |
| 550 | + if _access_check(cmd, mode): |
| 551 | + return cmd |
| 552 | + return None |
| 553 | + |
| 554 | + if path is None: |
| 555 | + path = os.environ.get("PATH", os.defpath) |
| 556 | + if not path: |
| 557 | + return None |
| 558 | + path = path.split(os.pathsep) |
| 559 | + |
| 560 | + if sys.platform == "win32": |
| 561 | + # The current directory takes precedence on Windows. |
| 562 | + if os.curdir not in path: |
| 563 | + path.insert(0, os.curdir) |
| 564 | + |
| 565 | + # PATHEXT is necessary to check on Windows. |
| 566 | + pathext = os.environ.get("PATHEXT", "").split(os.pathsep) |
| 567 | + # See if the given file matches any of the expected path extensions. |
| 568 | + # This will allow us to short circuit when given "python.exe". |
| 569 | + # If it does match, only test that one, otherwise we have to try |
| 570 | + # others. |
| 571 | + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): |
| 572 | + files = [cmd] |
| 573 | + else: |
| 574 | + files = [cmd + ext for ext in pathext] |
| 575 | + else: |
| 576 | + # On other platforms you don't have things like PATHEXT to tell you |
| 577 | + # what file suffixes are executable, so just pass on cmd as-is. |
| 578 | + files = [cmd] |
| 579 | + |
| 580 | + seen = set() |
| 581 | + for dir in path: |
| 582 | + normdir = os.path.normcase(dir) |
| 583 | + if normdir not in seen: |
| 584 | + seen.add(normdir) |
| 585 | + for thefile in files: |
| 586 | + name = os.path.join(dir, thefile) |
| 587 | + if _access_check(name, mode): |
| 588 | + return name |
| 589 | + return None |