Changeset 4da8e2d in trunk


Ignore:
Timestamp:
2020-07-16T18:42:09Z (5 years ago)
Author:
Itamar Turner-Trauring <itamar@…>
Branches:
master
Children:
37fa687
Parents:
7e7f771 (diff), 8d143af (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch '3342-base32-and-base62-to-python-3' into 3344.netstring-and-hashutil-to-python-3

Files:
5 added
11 edited

Legend:

Unmodified
Added
Removed
  • TabularUnified .gitignore

    r7e7f771 r4da8e2d  
    4545/coverage.xml
    4646/.hypothesis/
     47/eliot.log
     48/misc/python3/results.xml
     49/misc/python3/results.subunit2
    4750
    4851# This is the plaintext of the private environment needed for some CircleCI
  • TabularUnified .travis.yml

    r7e7f771 r4da8e2d  
    22language: python
    33cache: pip
    4 dist: trusty
     4dist: xenial
    55before_cache:
    66  - rm -f $HOME/.cache/pip/log/debug.log
     
    1717  - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py; fi
    1818  - pip list
    19   - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then pip install --user --upgrade codecov tox setuptools; fi
    20   - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then pip install --upgrade codecov tox setuptools; fi
     19  - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then pip install --user --upgrade codecov tox setuptools virtualenv; fi
     20  - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then pip install --upgrade codecov tox setuptools virtualenv; fi
    2121  - echo $PATH; which python; which pip; which tox
    2222  - python misc/build_helpers/show-tool-versions.py
     
    2525  - |
    2626    set -eo pipefail
    27     if [ "${T}" = "py35" ]; then
    28       python3 -m compileall -f -x tahoe-depgraph.py .
    29     else
    30       tox -e ${T}
    31     fi
     27    tox -e ${T}
    3228    # To verify that the resultant PyInstaller-generated binary executes
    3329    # cleanly (i.e., that it terminates with an exit code of 0 and isn't
     
    7066    env: T=pyinstaller LANG=en_US.UTF-8
    7167    language: generic  # "python" is not available on OS-X
    72   # this is a "lint" job that checks for python3 compatibility
    7368  - os: linux
    74     python: '3.5'
    75     env: T=py35
     69    python: '3.6'
     70    env: T=py36
    7671
    7772  fast_finish: true
  • TabularUnified setup.py

    r7e7f771 r4da8e2d  
    142142
    143143i2p_requires = [
    144     # See the comment in tor_requires.
    145     "txi2p >= 0.3.2",
     144    # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10.
     145    # URL lookups are in PEP-508 (via https://stackoverflow.com/a/54794506).
     146    # Also see the comment in tor_requires.
     147    "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p",
    146148]
    147149
     
    354356      packages=find_packages('src') + ['allmydata.test.plugins'],
    355357      classifiers=trove_classifiers,
    356       python_requires="<3.0",
     358      # We support Python 2.7, and we're working on support for 3.6 (the
     359      # highest version that PyPy currently supports).
     360      python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <3.7",
    357361      install_requires=install_requires,
    358362      extras_require={
     
    383387              "beautifulsoup4",
    384388              "html5lib",
     389              "junitxml",
    385390          ] + tor_requires + i2p_requires,
    386391          "tor": tor_requires,
  • TabularUnified src/allmydata/test/common_util.py

    r7e7f771 r4da8e2d  
    11from __future__ import print_function
    22
    3 import os, signal, sys, time
     3import os, signal, time
    44from random import randrange
    55from six.moves import StringIO
     
    99from twisted.trial import unittest
    1010
    11 from allmydata.util import fileutil, log
    1211from ..util.assertutil import precondition
    1312from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
     
    9089
    9190
    92 class NonASCIIPathMixin(object):
    93     def mkdir_nonascii(self, dirpath):
    94         # Kludge to work around the fact that buildbot can't remove a directory tree that has
    95         # any non-ASCII directory names on Windows. (#1472)
    96         if sys.platform == "win32":
    97             def _cleanup():
    98                 try:
    99                     fileutil.rm_dir(dirpath)
    100                 finally:
    101                     if os.path.exists(dirpath):
    102                         msg = ("We were unable to delete a non-ASCII directory %r created by the test. "
    103                                "This is liable to cause failures on future builds." % (dirpath,))
    104                         print(msg)
    105                         log.err(msg)
    106             self.addCleanup(_cleanup)
    107         os.mkdir(dirpath)
    108 
    109     def unicode_or_fallback(self, unicode_name, fallback_name, io_as_well=False):
    110         if not unicode_platform():
    111             try:
    112                 unicode_name.encode(get_filesystem_encoding())
    113             except UnicodeEncodeError:
    114                 return fallback_name
    115 
    116         if io_as_well:
    117             try:
    118                 unicode_name.encode(get_io_encoding())
    119             except UnicodeEncodeError:
    120                 return fallback_name
    121 
    122         return unicode_name
    123 
    124 
    12591class SignalMixin(object):
    12692    # This class is necessary for any code which wants to use Processes
  • TabularUnified src/allmydata/test/test_base62.py

    r7e7f771 r4da8e2d  
    2828    return bytes(list(map(random.randrange, [0]*n, [256]*n)))
    2929
    30 class T(unittest.TestCase):
     30class Base62(unittest.TestCase):
    3131    def _test_num_octets_that_encode_to_this_many_chars(self, chars, octets):
    3232        assert base62.num_octets_that_encode_to_this_many_chars(chars) == octets, "%s != %s <- %s" % (octets, base62.num_octets_that_encode_to_this_many_chars(chars), chars)
     
    3939        self.assertIsInstance(bs, bytes)
    4040        self.assertIsInstance(decoded, bytes)
     41        # Encoded string only uses values from the base62 allowed characters:
     42        self.assertFalse(set(encoded) - set(base62.chars))
    4143
    4244    @given(input_bytes=st.binary(max_size=100))
    4345    def test_roundtrip(self, input_bytes):
    4446        self._test_roundtrip(input_bytes)
     47
     48    def test_known_values(self):
     49        """Known values to ensure the algorithm hasn't changed."""
     50
     51        def check_expected(plaintext, encoded):
     52            result1 = base62.b2a(plaintext)
     53            self.assertEqual(encoded, result1)
     54            result2 = base62.a2b(encoded)
     55            self.assertEqual(plaintext, result2)
     56
     57        check_expected(b"hello", b'7tQLFHz')
     58        check_expected(b"", b'0')
     59        check_expected(b"zzz", b'0Xg7e')
     60        check_expected(b"\x36\xffWAT", b'49pq4mq')
     61        check_expected(b"1234 22323", b'1A0afZe9mxSZpz')
     62        check_expected(b"______", b'0TmAuCHJX')
    4563
    4664    def test_num_octets_that_encode_to_this_many_chars(self):
  • TabularUnified src/allmydata/test/test_client.py

    r7e7f771 r4da8e2d  
    8484              )
    8585
    86 class Basic(testutil.ReallyEqualMixin, testutil.NonASCIIPathMixin, unittest.TestCase):
     86class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
    8787    def test_loadable(self):
    8888        basedir = "test_client.Basic.test_loadable"
  • TabularUnified src/allmydata/util/base32.py

    r7e7f771 r4da8e2d  
    1212if PY2:
    1313    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
     14
     15if PY2:
     16    def backwardscompat_bytes(b):
     17        """
     18        Replace Future bytes with native Python 2 bytes, so % works
     19        consistently until other modules are ported.
     20        """
     21        return getattr(b, "__native__", lambda: b)()
     22else:
     23    def backwardscompat_bytes(b):
     24        return b
    1425
    1526import base64
     
    4758    return b''.join(_get_trailing_chars_without_lsbs(N, d=d))
    4859
    49 BASE32CHAR =b'['+get_trailing_chars_without_lsbs(0)+b']'
    50 BASE32CHAR_4bits =b'['+get_trailing_chars_without_lsbs(1)+b']'
    51 BASE32CHAR_3bits =b'['+get_trailing_chars_without_lsbs(2)+b']'
    52 BASE32CHAR_2bits =b'['+get_trailing_chars_without_lsbs(3)+b']'
    53 BASE32CHAR_1bits =b'['+get_trailing_chars_without_lsbs(4)+b']'
    54 BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits
    55 BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits
    56 BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits
    57 BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits
    58 BASE32STR_anybytes =b'((?:%s{8})*' % (BASE32CHAR,) + b"(?:|%s|%s|%s|%s))" % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes)
     60BASE32CHAR = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(0)+b']')
     61BASE32CHAR_4bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(1)+b']')
     62BASE32CHAR_3bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(2)+b']')
     63BASE32CHAR_2bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(3)+b']')
     64BASE32CHAR_1bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(4)+b']')
     65BASE32STR_1byte = backwardscompat_bytes(BASE32CHAR+BASE32CHAR_3bits)
     66BASE32STR_2bytes = backwardscompat_bytes(BASE32CHAR+b'{3}'+BASE32CHAR_1bits)
     67BASE32STR_3bytes = backwardscompat_bytes(BASE32CHAR+b'{4}'+BASE32CHAR_4bits)
     68BASE32STR_4bytes = backwardscompat_bytes(BASE32CHAR+b'{6}'+BASE32CHAR_2bits)
     69BASE32STR_anybytes = backwardscompat_bytes(bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes))
    5970
    6071def b2a(os):
     
    8495NUM_QS_TO_NUM_OS=(0, 1, 1, 2, 2, 3, 3, 4)
    8596NUM_QS_LEGIT=(1, 0, 1, 0, 1, 1, 0, 1,)
    86 NUM_QS_TO_NUM_BITS=tuple([x*8 for x in NUM_QS_TO_NUM_OS])
     97NUM_QS_TO_NUM_BITS=tuple([_x*8 for _x in NUM_QS_TO_NUM_OS])
     98if PY2:
     99    del _x
    87100
    88101# A fast way to determine whether a given string *could* be base-32 encoded data, assuming that the
  • TabularUnified src/allmydata/util/fileutil.py

    r7e7f771 r4da8e2d  
    1313    from ctypes import WINFUNCTYPE, WinError, windll, POINTER, byref, c_ulonglong, \
    1414        create_unicode_buffer, get_last_error
    15     from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPWSTR, LPVOID, HANDLE
     15    from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPWSTR, LPVOID
    1616
    1717from twisted.python import log
     
    537537        log.msg("OS call to get disk statistics failed")
    538538        return 0
    539 
    540 
    541 if sys.platform == "win32":
    542     # <http://msdn.microsoft.com/en-us/library/aa363858%28v=vs.85%29.aspx>
    543     CreateFileW = WINFUNCTYPE(
    544         HANDLE,  LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE,
    545         use_last_error=True
    546     )(("CreateFileW", windll.kernel32))
    547 
    548     GENERIC_WRITE        = 0x40000000
    549     FILE_SHARE_READ      = 0x00000001
    550     FILE_SHARE_WRITE     = 0x00000002
    551     OPEN_EXISTING        = 3
    552     INVALID_HANDLE_VALUE = 0xFFFFFFFF
    553 
    554     # <http://msdn.microsoft.com/en-us/library/aa364439%28v=vs.85%29.aspx>
    555     FlushFileBuffers = WINFUNCTYPE(
    556         BOOL,  HANDLE,
    557         use_last_error=True
    558     )(("FlushFileBuffers", windll.kernel32))
    559 
    560     # <http://msdn.microsoft.com/en-us/library/ms724211%28v=vs.85%29.aspx>
    561     CloseHandle = WINFUNCTYPE(
    562         BOOL,  HANDLE,
    563         use_last_error=True
    564     )(("CloseHandle", windll.kernel32))
    565 
    566     # <http://social.msdn.microsoft.com/forums/en-US/netfxbcl/thread/4465cafb-f4ed-434f-89d8-c85ced6ffaa8/>
    567     def flush_volume(path):
    568         abspath = os.path.realpath(path)
    569         if abspath.startswith("\\\\?\\"):
    570             abspath = abspath[4 :]
    571         drive = os.path.splitdrive(abspath)[0]
    572 
    573         print("flushing %r" % (drive,))
    574         hVolume = CreateFileW(u"\\\\.\\" + drive,
    575                               GENERIC_WRITE,
    576                               FILE_SHARE_READ | FILE_SHARE_WRITE,
    577                               None,
    578                               OPEN_EXISTING,
    579                               0,
    580                               None
    581                              )
    582         if hVolume == INVALID_HANDLE_VALUE:
    583             raise WinError(get_last_error())
    584 
    585         if FlushFileBuffers(hVolume) == 0:
    586             raise WinError(get_last_error())
    587 
    588         CloseHandle(hVolume)
    589 else:
    590     def flush_volume(path):
    591         # use sync()?
    592         pass
    593539
    594540
  • TabularUnified src/allmydata/util/rrefutil.py

    r7e7f771 r4da8e2d  
    11
    22from twisted.internet import address
    3 from foolscap.api import Violation, RemoteException, DeadReferenceError, \
    4      SturdyRef
     3from foolscap.api import Violation, RemoteException, SturdyRef
     4
    55
    66def add_version_to_remote_reference(rref, default):
     
    1919    d.addCallbacks(_got_version, _no_get_version)
    2020    return d
    21 
    22 def trap_and_discard(f, *errorTypes):
    23     f.trap(*errorTypes)
    24 
    25 def trap_deadref(f):
    26     return trap_and_discard(f, DeadReferenceError)
    2721
    2822
  • TabularUnified src/allmydata/version_checks.py

    r7e7f771 r4da8e2d  
    298298def _get_package_versions_and_locations():
    299299    import warnings
    300     from _auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \
     300    from ._auto_deps import package_imports, global_deprecation_messages, deprecation_messages, \
    301301        runtime_warning_messages, warning_imports, ignorable
    302302
  • TabularUnified tox.ini

    r7e7f771 r4da8e2d  
    88
    99[tox]
    10 envlist = {py27,pypy27}{-coverage,}
     10envlist = {py27,pypy27,py36}{-coverage,}
    1111minversion = 2.4
    1212
     
    4646extras = test
    4747commands =
     48         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
    4849         tahoe --version
    49          trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
     50
     51[testenv:py36]
     52# git inside of ratchet.sh needs $HOME.
     53passenv = HOME
     54commands = {toxinidir}/misc/python3/ratchet.sh
    5055
    5156[testenv:integration]
Note: See TracChangeset for help on using the changeset viewer.