Changeset 6e24def in trunk

Timestamp:

2020-08-13T19:53:02Z (5 years ago)

Author:

Itamar Turner-Trauring <itamar@…>

Branches:

master

Children:

af61571

Parents:

1037854 (diff), 942c5d52 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch '3376.encodingutil-python-3' into 3377.configutil-connection_status-python-3

Files:

• .circleci/Dockerfile.centos (modified) (3 diffs)
• .circleci/Dockerfile.debian (modified) (3 diffs)
• .circleci/Dockerfile.fedora (modified) (3 diffs)
• .circleci/Dockerfile.slackware (deleted)
• .circleci/Dockerfile.ubuntu (modified) (3 diffs)
• .circleci/config.yml (modified) (20 diffs)
• .circleci/populate-wheelhouse.sh (modified) (1 diff)
• .circleci/run-tests.sh (modified) (1 diff)
• .travis.yml (deleted)
• misc/python3/ratchet-passing (modified) (3 diffs)
• misc/python3/ratchet.sh (modified) (2 diffs)
• newsfragments/3323.removed (added)
• newsfragments/3336.minor (added)
• newsfragments/3358.minor (added)
• newsfragments/3370.minor (added)
• newsfragments/3376.minor (added)
• src/allmydata/client.py (modified) (1 diff)
• src/allmydata/dirnode.py (modified) (3 diffs)
• src/allmydata/immutable/happiness_upload.py (modified) (8 diffs)
• src/allmydata/test/common_py3.py (modified) (3 diffs)
• src/allmydata/test/common_util.py (modified) (3 diffs)
• src/allmydata/test/test_encodingutil.py (modified) (21 diffs)
• src/allmydata/test/test_happiness.py (modified) (1 diff)
• src/allmydata/test/test_util.py (modified) (13 diffs)
• src/allmydata/util/_python3.py (modified) (4 diffs)
• src/allmydata/util/encodingutil.py (modified) (15 diffs)
• src/allmydata/util/fileutil.py (modified) (6 diffs)
• src/allmydata/util/statistics.py (modified) (1 diff)
• tox.ini (modified) (1 diff)

.circleci/Dockerfile.centos ¶

@@ -1 +1 @@
 ARG TAG
 FROM centos:${TAG}
+ARG PYTHON_VERSION
 
 ENV WHEELHOUSE_PATH /tmp/wheelhouse
@@ -12 +13 @@
     sudo \
     make automake gcc gcc-c++ \
-    python2 \
-    python2-devel \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-devel \
     libffi-devel \
     openssl-devel \
@@ -24 +25 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}"

.circleci/Dockerfile.debian ¶

@@ -1 +1 @@
 ARG TAG
 FROM debian:${TAG}
+ARG PYTHON_VERSION
 
 ENV WHEELHOUSE_PATH /tmp/wheelhouse
@@ -9 +10 @@
 RUN apt-get --quiet update && \
     apt-get --quiet --yes install \
-        git \
-        lsb-release \
+        git \
+        lsb-release \
         sudo \
-        build-essential \
-        python2.7 \
-        python2.7-dev \
-        libffi-dev \
-        libssl-dev \
-        libyaml-dev \
-        virtualenv
+        build-essential \
+        python${PYTHON_VERSION} \
+        python${PYTHON_VERSION}-dev \
+        libffi-dev \
+        libssl-dev \
+        libyaml-dev \
+        virtualenv
 
 # Get the project source.  This is better than it seems.  CircleCI will
@@ -24 +25 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}"
 
 # Only the integration tests currently need this but it doesn't hurt to always

.circleci/Dockerfile.fedora ¶

@@ -1 +1 @@
 ARG TAG
 FROM fedora:${TAG}
+ARG PYTHON_VERSION
 
 ENV WHEELHOUSE_PATH /tmp/wheelhouse
@@ -12 +13 @@
     sudo \
     make automake gcc gcc-c++ \
-    python \
-    python-devel \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-devel \
     libffi-devel \
     openssl-devel \
@@ -24 +25 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}"

.circleci/Dockerfile.ubuntu ¶

@@ -1 +1 @@
 ARG TAG
 FROM ubuntu:${TAG}
+ARG PYTHON_VERSION
 
 ENV WHEELHOUSE_PATH /tmp/wheelhouse
@@ -14 +15 @@
         sudo \
         build-essential \
-        python2.7 \
-        python2.7-dev \
+        python${PYTHON_VERSION} \
+        python${PYTHON_VERSION}-dev \
         libffi-dev \
         libssl-dev \
@@ -27 +28 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}"

.circleci/config.yml ¶

@@ -27 +27 @@
       - "centos-8"
 
-      - "slackware-14.2"
-
       - "nixos-19.09"
 
       # Test against PyPy 2.7
       - "pypy2.7-buster"
+
+      # Just one Python 3.6 configuration while the port is in-progress.
+      - "python3.6"
 
       # Other assorted tasks and configurations
@@ -73 +74 @@
       - "build-image-fedora-29"
       - "build-image-centos-8"
-      - "build-image-slackware-14.2"
       - "build-image-pypy-2.7-buster"
+      - "build-image-python36-ubuntu"
 
 
@@ -122 +123 @@
   debian-9: &DEBIAN
     docker:
-      - image: "tahoelafsci/debian:9"
+      - image: "tahoelafsci/debian:9-py2.7"
         user: "nobody"
 
@@ -199 +200 @@
     <<: *DEBIAN
     docker:
-      - image: "tahoelafsci/debian:8"
+      - image: "tahoelafsci/debian:8-py2.7"
         user: "nobody"
 
@@ -206 +207 @@
     <<: *DEBIAN
     docker:
-      - image: "tahoelafsci/pypy:2.7-buster"
+      - image: "tahoelafsci/pypy:buster-py2"
         user: "nobody"
 
@@ -262 +263 @@
     <<: *DEBIAN
     docker:
-      - image: "tahoelafsci/ubuntu:16.04"
-        user: "nobody"
-
-
-  ubuntu-18.04:
-    <<: *DEBIAN
-    docker:
-      - image: "tahoelafsci/ubuntu:18.04"
-        user: "nobody"
+      - image: "tahoelafsci/ubuntu:16.04-py2.7"
+        user: "nobody"
+
+
+  ubuntu-18.04: &UBUNTU_18_04
+    <<: *DEBIAN
+    docker:
+      - image: "tahoelafsci/ubuntu:18.04-py2.7"
+        user: "nobody"
+
+
+  python3.6:
+    <<: *UBUNTU_18_04
+    docker:
+      - image: "tahoelafsci/ubuntu:18.04-py3"
+        user: "nobody"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
 
 
@@ -282 +294 @@
   centos-8: &RHEL_DERIV
     docker:
-      - image: "tahoelafsci/centos:8"
+      - image: "tahoelafsci/centos:8-py2"
         user: "nobody"
 
@@ -304 +316 @@
     <<: *RHEL_DERIV
     docker:
-      - image: "tahoelafsci/fedora:28"
+      - image: "tahoelafsci/fedora:28-py"
         user: "nobody"
 
@@ -311 +323 @@
     <<: *RHEL_DERIV
     docker:
-      - image: "tahoelafsci/fedora:29"
-        user: "nobody"
-
-
-  slackware-14.2:
-    docker:
-      - image: "tahoelafsci/slackware:14.2"
-        user: "nobody"
-
-    environment: *UTF_8_ENVIRONMENT
-
-    # pip cannot install packages if the working directory is not readable.
-    # We want to run a lot of steps as nobody instead of as root.
-    working_directory: "/tmp/project"
-
-    steps:
-      - "checkout"
-      - run: *SETUP_VIRTUALENV
-      - run: *RUN_TESTS
-      - store_test_results: *STORE_TEST_RESULTS
-      - store_artifacts: *STORE_TEST_LOG
-      - store_artifacts: *STORE_OTHER_ARTIFACTS
-      - run: *SUBMIT_COVERAGE
+      - image: "tahoelafsci/fedora:29-py"
+        user: "nobody"
+
 
   nixos-19.09:
@@ -398 +390 @@
 
     environment:
-      DISTRO: "tahoelafsci/<DISTRO>:foo"
-      TAG: "tahoelafsci/distro:<TAG>"
+      DISTRO: "tahoelafsci/<DISTRO>:foo-py2"
+      TAG: "tahoelafsci/distro:<TAG>-py2"
+      PYTHON_VERSION: "tahoelafsci/distro:tag-py<PYTHON_VERSION}"
 
     steps:
@@ -451 +444 @@
                 build \
                 --build-arg TAG=${TAG} \
-                -t tahoelafsci/${DISTRO}:${TAG} \
+                --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
+                -t tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION} \
                 -f ~/project/.circleci/Dockerfile.${DISTRO} \
                 ~/project/
@@ -457 +451 @@
           name: "Push image"
           command: |
-            docker push tahoelafsci/${DISTRO}:${TAG}
+            docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
 
 
@@ -466 +460 @@
       DISTRO: "debian"
       TAG: "8"
+      PYTHON_VERSION: "2.7"
 
 
@@ -474 +469 @@
       DISTRO: "debian"
       TAG: "9"
+      PYTHON_VERSION: "2.7"
 
 
@@ -482 +478 @@
       DISTRO: "ubuntu"
       TAG: "16.04"
+      PYTHON_VERSION: "2.7"
 
 
@@ -490 +487 @@
       DISTRO: "ubuntu"
       TAG: "18.04"
+      PYTHON_VERSION: "2.7"
+
+
+  build-image-python36-ubuntu:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "18.04"
+      PYTHON_VERSION: "3"
 
 
@@ -506 +513 @@
       DISTRO: "centos"
       TAG: "8"
+      PYTHON_VERSION: "2"
 
 
@@ -514 +522 @@
       DISTRO: "fedora"
       TAG: "28"
+      # The default on Fedora (this version anyway) is still Python 2.
+      PYTHON_VERSION: ""
 
 
@@ -524 +534 @@
 
 
-  build-image-slackware-14.2:
-    <<: *BUILD_IMAGE
-
-    environment:
-      DISTRO: "slackware"
-      TAG: "14.2"
-
-
   build-image-pypy-2.7-buster:
     <<: *BUILD_IMAGE
@@ -537 +539 @@
     environment:
       DISTRO: "pypy"
-      TAG: "2.7-buster"
+      TAG: "buster"
+      # We only have Python 2 for PyPy right now so there's no support for
+      # setting up PyPy 3 in the image building toolchain.  This value is just
+      # for constructing the right Docker image tag.
+      PYTHON_VERSION: "2"

.circleci/populate-wheelhouse.sh ¶

@@ -37 +37 @@
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 
-# Populate the wheelhouse, if necessary.
-"${PIP}" \
+# Populate the wheelhouse, if necessary.  zfec 1.5.3 can only be built with a
+# UTF-8 environment so make sure we have one, at least for this invocation.
+LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \

.circleci/run-tests.sh ¶

@@ -82 +82 @@
 
 if [ -n "${ARTIFACTS}" ]; then
+    if [ ! -e "${SUBUNIT2}" ]; then
+        echo "subunitv2 output file does not exist: ${SUBUNIT2}"
+        exit 1
+    fi
+
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+    # Always succeed even if subunit2junitxml fails.  subunit2junitxml signals
+    # failure if the stream it is processing contains test failures.  This is
+    # not what we care about.  If we cared about it, the test command above
+    # would have signalled failure already and we wouldn't be here.
+    "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || true
 fi

misc/python3/ratchet-passing ¶

@@ -10 +10 @@
 allmydata.test.test_abbreviate.Abbreviate.test_space
 allmydata.test.test_abbreviate.Abbreviate.test_time
+allmydata.test.test_backupdb.BackupDB.test_basic
+allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2
+allmydata.test.test_backupdb.BackupDB.test_wrong_version
 allmydata.test.test_base32.Base32.test_a2b
 allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons
@@ -71 +74 @@
 allmydata.test.test_dictutil.DictUtil.test_auxdict
 allmydata.test.test_dictutil.DictUtil.test_dict_of_sets
+allmydata.test.test_encodingutil.EncodingUtilErrors.test_argv_to_unicode
+allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding
+allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding_not_from_stdout
+allmydata.test.test_encodingutil.EncodingUtilErrors.test_no_unicode_normalization
+allmydata.test.test_encodingutil.EncodingUtilErrors.test_unicode_to_output
+allmydata.test.test_encodingutil.FilePaths.test_extend_filepath
+allmydata.test.test_encodingutil.FilePaths.test_to_filepath
+allmydata.test.test_encodingutil.FilePaths.test_unicode_from_filepath
+allmydata.test.test_encodingutil.FilePaths.test_unicode_segments_from
+allmydata.test.test_encodingutil.MacOSXLeopard.test_argv_to_unicode
+allmydata.test.test_encodingutil.MacOSXLeopard.test_listdir_unicode
+allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_platform_py3
+allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_argv_py3
+allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_output
+allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_url
+allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_argv_to_unicode
+allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_listdir_unicode
+allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_platform_py3
+allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_argv_py3
+allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_output
+allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_url
+allmydata.test.test_encodingutil.OpenBSD.test_argv_to_unicode
+allmydata.test.test_encodingutil.OpenBSD.test_listdir_unicode
+allmydata.test.test_encodingutil.OpenBSD.test_unicode_platform_py3
+allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_argv_py3
+allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_output
+allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_url
+allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_ascii
+allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_default
+allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_latin1
+allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_utf8
+allmydata.test.test_encodingutil.QuotePaths.test_quote_filepath
+allmydata.test.test_encodingutil.QuotePaths.test_quote_path
+allmydata.test.test_encodingutil.StdlibUnicode.test_mkdir_open_exists_abspath_listdir_expanduser
+allmydata.test.test_encodingutil.TestToFromStr.test_from_utf8_or_none
+allmydata.test.test_encodingutil.TestToFromStr.test_to_str
+allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_argv_to_unicode
+allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_listdir_unicode
+allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_platform_py3
+allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_argv_py3
+allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_output
+allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_url
+allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_argv_to_unicode
+allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_listdir_unicode
+allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_platform_py3
+allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_argv_py3
+allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_output
+allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_url
+allmydata.test.test_encodingutil.Windows.test_argv_to_unicode
+allmydata.test.test_encodingutil.Windows.test_unicode_platform_py3
+allmydata.test.test_encodingutil.Windows.test_unicode_to_argv_py3
+allmydata.test.test_encodingutil.Windows.test_unicode_to_output
+allmydata.test.test_encodingutil.Windows.test_unicode_to_url
+allmydata.test.test_happiness.Happiness.test_100
+allmydata.test.test_happiness.Happiness.test_calc_happy
+allmydata.test.test_happiness.Happiness.test_everything_broken
+allmydata.test.test_happiness.Happiness.test_hypothesis0
+allmydata.test.test_happiness.Happiness.test_hypothesis_0
+allmydata.test.test_happiness.Happiness.test_hypothesis_1
+allmydata.test.test_happiness.Happiness.test_placement_1
+allmydata.test.test_happiness.Happiness.test_placement_simple
+allmydata.test.test_happiness.Happiness.test_redistribute
+allmydata.test.test_happiness.Happiness.test_unhappy
+allmydata.test.test_happiness.HappinessUtils.test_residual_0
+allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph
+allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph
+allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy
+allmydata.test.test_happiness.PlacementTests.test_more_hypothesis
 allmydata.test.test_hashtree.Complete.test_create
 allmydata.test.test_hashtree.Complete.test_dump
@@ -144 +215 @@
 allmydata.test.test_time_format.TimeFormat.test_parse_date
 allmydata.test.test_time_format.TimeFormat.test_parse_duration
+allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode
+allmydata.test.test_util.FileUtil.test_create_long_path
+allmydata.test.test_util.FileUtil.test_disk_stats
+allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative
+allmydata.test.test_util.FileUtil.test_du
+allmydata.test.test_util.FileUtil.test_encrypted_tempfile
+allmydata.test.test_util.FileUtil.test_get_pathinfo
+allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink
+allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode
+allmydata.test.test_util.FileUtil.test_remove_if_possible
+allmydata.test.test_util.FileUtil.test_rename
+allmydata.test.test_util.FileUtil.test_rename_no_overwrite
+allmydata.test.test_util.FileUtil.test_replace_file
+allmydata.test.test_util.FileUtil.test_rm_dir
+allmydata.test.test_util.FileUtil.test_windows_expanduser_win7
+allmydata.test.test_util.FileUtil.test_windows_expanduser_xp
+allmydata.test.test_util.FileUtil.test_write_atomically
+allmydata.test.test_util.IDLib.test_nodeid_b2a
+allmydata.test.test_util.Math.test_round_sigfigs
+allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True
+allmydata.test.test_util.PollMixinTests.test_PollMixin_True
+allmydata.test.test_util.PollMixinTests.test_timeout
+allmydata.test.test_util.YAML.test_convert
 allmydata.test.test_version.CheckRequirement.test_cross_check
 allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions

misc/python3/ratchet.sh ¶

@@ -10 +10 @@
 cd "../.."
 
-# Since both of the next calls are expected to exit non-0, relax our guard.
-set +e
-SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter subunitv2-file allmydata
-subunit2junitxml < "$base/results.subunit2" > "$base/results.xml"
-set -e
+export SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2"
+# Since the next two calls are expected to exit non-0, relax our guard.
+trial --reporter=subunitv2-file allmydata || true
+subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" || true
 
 # Okay, now we're clear.
@@ -33 +32 @@
   export TERM=ansi
 fi
-git diff "$tracking_filename"
 
-exit $code
+echo "The ${tracking_filename} diff is:"
+echo "================================="
+# "git diff" gets pretty confused in this execution context when trying to
+# write to stdout.  Somehow it fails with SIGTTOU.
+git diff -- "${tracking_filename}" > tracking.diff
+cat tracking.diff
+echo "================================="
+
+echo "Exiting with code ${code} from ratchet.py."
+exit ${code}

src/allmydata/client.py ¶

@@ -742 +742 @@
         private_key, public_key = ed25519.signing_keypair_from_string(private_key_str)
         public_key_str = ed25519.string_from_verifying_key(public_key)
-        self.config.write_config_file("node.pubkey", public_key_str + "\n")
+        self.config.write_config_file("node.pubkey", public_key_str + "\n", "w")
         self._node_private_key = private_key
         self._node_public_key = public_key

src/allmydata/dirnode.py ¶

@@ -1 +1 @@
 """Directory Node implementation."""
-import time, unicodedata
+import time
 
 from zope.interface import implementer
@@ -19 +19 @@
 from allmydata.monitor import Monitor
 from allmydata.util import hashutil, base32, log
-from allmydata.util.encodingutil import quote_output
+from allmydata.util.encodingutil import quote_output, normalize
 from allmydata.util.assertutil import precondition
 from allmydata.util.netstring import netstring, split_netstring
@@ -101 +101 @@
     return metadata
 
-
-# 'x' at the end of a variable name indicates that it holds a Unicode string that may not
-# be NFC-normalized.
-
-def normalize(namex):
-    return unicodedata.normalize('NFC', namex)
 
 # TODO: {Deleter,MetadataSetter,Adder}.modify all start by unpacking the

src/allmydata/immutable/happiness_upload.py ¶

@@ -1 @@
-
-from Queue import PriorityQueue
+"""
+Algorithms for figuring out happiness, the number of unique nodes the data is
+on.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # We omit dict, just in case newdict breaks things for external Python 2 code.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
+
+from queue import PriorityQueue
 
 
@@ -36 +51 @@
     # BLACK vertices are those we have seen and explored
     BLACK = 2
-    color        = [WHITE for i in xrange(len(graph))]
-    predecessor  = [None for i in xrange(len(graph))]
-    distance     = [-1 for i in xrange(len(graph))]
+    color        = [WHITE for i in range(len(graph))]
+    predecessor  = [None for i in range(len(graph))]
+    distance     = [-1 for i in range(len(graph))]
     queue = [s] # vertices that we haven't explored yet.
     color[s] = GRAY
@@ -59 +74 @@
     flow network in adjacency-list form, and f is a flow in graph.
     """
-    new_graph = [[] for i in xrange(len(graph))]
-    cf = [[0 for s in xrange(len(graph))] for sh in xrange(len(graph))]
-    for i in xrange(len(graph)):
+    new_graph = [[] for i in range(len(graph))]
+    cf = [[0 for s in range(len(graph))] for sh in range(len(graph))]
+    for i in range(len(graph)):
         for v in graph[i]:
             if f[i][v] == 1:
@@ -136 +151 @@
 
     dim = len(graph)
-    flow_function = [[0 for sh in xrange(dim)] for s in xrange(dim)]
+    flow_function = [[0 for sh in range(dim)] for s in range(dim)]
     residual_graph, residual_function = residual_network(graph, flow_function)
 
@@ -261 +276 @@
     #print "servermap %s" % servermap
     for peer in peers:
-        if servermap.has_key(peer):
+        if peer in servermap:
             for s in servermap[peer]:
-                if share_to_index.has_key(s):
+                if s in share_to_index:
                     indexedShares.append(share_to_index[s])
         graph.insert(peer_to_index[peer], indexedShares)
@@ -374 +389 @@
     #print "new_peers %s" % new_peers
     #print "new_mappings %s" % new_mappings
-    mappings = dict(readonly_mappings.items() + existing_mappings.items() + new_mappings.items())
+    mappings = dict(list(readonly_mappings.items()) + list(existing_mappings.items()) + list(new_mappings.items()))
     homeless_shares = set()
     for share in mappings:
@@ -385 +400 @@
             {
                 k: v
-                for k, v in peers_to_shares.items()
+                for k, v in list(peers_to_shares.items())
                 if k not in readonly_peers
             }
@@ -402 +417 @@
     return {
         k: v.pop() if v else next(peer_iter)
-        for k, v in mappings.items()
+        for k, v in list(mappings.items())
     }

src/allmydata/test/common_py3.py ¶

@@ -14 +14 @@
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+from past.builtins import unicode
+
 import os
 import time
@@ -19 +21 @@
 
 from twisted.internet import reactor
+from twisted.trial import unittest
+
+from ..util.assertutil import precondition
+from ..util.encodingutil import unicode_platform, get_filesystem_encoding
 
 
@@ -66 +72 @@
             signal.signal(signal.SIGCHLD, self.sigchldHandler)
         return super(SignalMixin, self).tearDown()
+
+
+class ReallyEqualMixin(object):
+    def failUnlessReallyEqual(self, a, b, msg=None):
+        self.assertEqual(a, b, msg)
+        self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg))
+
+
+def skip_if_cannot_represent_filename(u):
+    precondition(isinstance(u, unicode))
+
+    enc = get_filesystem_encoding()
+    if not unicode_platform():
+        try:
+            u.encode(enc)
+        except UnicodeEncodeError:
+            raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")

src/allmydata/test/common_util.py ¶

@@ -10 +10 @@
 
 from ..util.assertutil import precondition
-from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
-                                         get_io_encoding)
-from ..scripts import runner
-from .common_py3 import SignalMixin
+from allmydata.util.encodingutil import get_io_encoding
+from future.utils import PY2
+if PY2: # XXX this is a hack that makes some tests pass on Python3, remove
+        # in the future
+    from ..scripts import runner
+# Imported for backwards compatibility:
+from .common_py3 import (
+    SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin,
+)
 
-
-def skip_if_cannot_represent_filename(u):
-    precondition(isinstance(u, unicode))
-
-    enc = get_filesystem_encoding()
-    if not unicode_platform():
-        try:
-            u.encode(enc)
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
 
 def skip_if_cannot_represent_argv(u):
@@ -83 +78 @@
     assert result != s, "Internal error -- flip_one_bit() produced the same string as its input: %s == %s" % (result, s)
     return result
-
-
-class ReallyEqualMixin(object):
-    def failUnlessReallyEqual(self, a, b, msg=None):
-        self.assertEqual(a, b, msg)
-        self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg))
 
 
@@ -184 +173 @@
     make_readonly = _make_readonly
     make_accessible = _make_accessible
+
+
+__all__ = [
+    "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin",
+    "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli",
+    "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit",
+    "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin"
+]

src/allmydata/test/test_encodingutil.py ¶

@@ -1 +1 @@
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2, PY3
+if PY2:
+    # We don't import str because omg way too ambiguous in this context.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+
+from past.builtins import unicode
 
 lumiere_nfc = u"lumi\u00E8re"
@@ -44 +54 @@
             open(os.path.join(tmpdir, fname), 'w').close()
 
-        # Use Unicode API under Windows or MacOS X
-        if sys.platform in ('win32', 'darwin'):
+        # On Python 2, listing directories returns unicode under Windows or
+        # MacOS X if the input is unicode. On Python 3, it always returns
+        # Unicode.
+        if PY2 and sys.platform in ('win32', 'darwin'):
             dirlist = os.listdir(unicode(tmpdir))
         else:
@@ -60 +72 @@
 
 import os, sys, locale
+from unittest import skipIf
 
 from twisted.trial import unittest
@@ -65 +78 @@
 from twisted.python.filepath import FilePath
 
-from allmydata.test.common_util import ReallyEqualMixin
+from allmydata.test.common_py3 import (
+    ReallyEqualMixin, skip_if_cannot_represent_filename,
+)
 from allmydata.util import encodingutil, fileutil
 from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \
@@ -71 +86 @@
     quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \
     get_io_encoding, get_filesystem_encoding, to_str, from_utf8_or_none, _reload, \
-    to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from
-from allmydata.dirnode import normalize
-from .common_util import skip_if_cannot_represent_filename
+    to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \
+    unicode_to_argv
 from twisted.python import usage
 
@@ -91 +105 @@
         mock_stdout.encoding = 'cp65001'
         _reload()
-        self.failUnlessReallyEqual(get_io_encoding(), 'utf-8')
+        self.assertEqual(get_io_encoding(), 'utf-8')
 
         mock_stdout.encoding = 'koi8-r'
@@ -123 +137 @@
         preferredencoding = None
         _reload()
-        self.failUnlessReallyEqual(get_io_encoding(), 'utf-8')
+        self.assertEqual(get_io_encoding(), 'utf-8')
 
     def test_argv_to_unicode(self):
@@ -151 +165 @@
 # Unicode entities on the filesystem.
 class EncodingUtilNonUnicodePlatform(unittest.TestCase):
+    @skipIf(PY3, "Python 3 is always Unicode, regardless of OS.")
     def setUp(self):
         # Mock sys.platform because unicode_platform() uses it
@@ -212 +227 @@
 
     def test_unicode_to_url(self):
-        self.failUnless(unicode_to_url(lumiere_nfc), "lumi\xc3\xa8re")
+        self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re")
 
     def test_unicode_to_output(self):
@@ -225 +240 @@
         self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.argv)
 
-    def test_unicode_platform(self):
+    @skipIf(PY3, "Python 2 only.")
+    def test_unicode_to_argv_py2(self):
+        """unicode_to_argv() converts to bytes on Python 2."""
+        self.assertEqual(unicode_to_argv("abc"), u"abc".encode(self.io_encoding))
+
+    @skipIf(PY2, "Python 3 only.")
+    def test_unicode_to_argv_py3(self):
+        """unicode_to_argv() is noop on Python 3."""
+        self.assertEqual(unicode_to_argv("abc"), "abc")
+
+    @skipIf(PY3, "Python 3 only.")
+    def test_unicode_platform_py2(self):
         matrix = {
           'linux2': False,
@@ -237 +263 @@
         self.failUnlessReallyEqual(unicode_platform(), matrix[self.platform])
 
+    @skipIf(PY2, "Python 3 isn't Python 2.")
+    def test_unicode_platform_py3(self):
+        _reload()
+        self.failUnlessReallyEqual(unicode_platform(), True)
+
     def test_listdir_unicode(self):
         if 'dirlist' not in dir(self):
@@ -249 +280 @@
 
         def call_os_listdir(path):
-            return self.dirlist
+            if PY2:
+                return self.dirlist
+            else:
+                # Python 3 always lists unicode filenames:
+                return [d.decode(self.filesystem_encoding) if isinstance(d, bytes)
+                        else d
+                        for d in self.dirlist]
+    
         self.patch(os, 'listdir', call_os_listdir)
 
@@ -259 +297 @@
         filenames = listdir_unicode(u'/dummy')
 
-        self.failUnlessEqual(set([normalize(fname) for fname in filenames]),
+        self.failUnlessEqual(set([encodingutil.normalize(fname) for fname in filenames]),
                              set(TEST_FILENAMES))
 
@@ -279 +317 @@
         open(fn, 'wb').close()
         self.failUnless(os.path.exists(fn))
-        self.failUnless(os.path.exists(os.path.join(os.getcwdu(), fn)))
+        if PY2:
+            getcwdu = os.getcwdu
+        else:
+            getcwdu = os.getcwd
+        self.failUnless(os.path.exists(os.path.join(getcwdu(), fn)))
         filenames = listdir_unicode(lumiere_nfc)
 
         # We only require that the listing includes a filename that is canonically equivalent
         # to lumiere_nfc (on Mac OS X, it will be the NFD equivalent).
-        self.failUnlessIn(lumiere_nfc + ".txt", set([normalize(fname) for fname in filenames]))
+        self.failUnlessIn(lumiere_nfc + u".txt", set([encodingutil.normalize(fname) for fname in filenames]))
 
         expanded = fileutil.expanduser(u"~/" + lumiere_nfc)
@@ -315 +357 @@
         if out[0:2] == 'b"':
             pass
-        elif isinstance(inp, str):
-            self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quote_newlines=quote_newlines), out)
-            self.failUnlessReallyEqual(quote_output(unicode(inp), encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2)
+        elif isinstance(inp, bytes):
+            try:
+                unicode_inp = inp.decode("utf-8")
+            except UnicodeDecodeError:
+                # Some things decode on Python 2, but not Python 3...
+                return
+            self.failUnlessReallyEqual(quote_output(unicode_inp, encoding=enc, quote_newlines=quote_newlines), out)
+            self.failUnlessReallyEqual(quote_output(unicode_inp, encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2)
         else:
-            self.failUnlessReallyEqual(quote_output(inp.encode('utf-8'), encoding=enc, quote_newlines=quote_newlines), out)
-            self.failUnlessReallyEqual(quote_output(inp.encode('utf-8'), encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2)
+            try:
+                bytes_inp = inp.encode('utf-8')
+            except UnicodeEncodeError:
+                # Some things encode on Python 2, but not Python 3, e.g.
+                # surrogates like u"\uDC00\uD800"...
+                return
+            self.failUnlessReallyEqual(quote_output(bytes_inp, encoding=enc, quote_newlines=quote_newlines), out)
+            self.failUnlessReallyEqual(quote_output(bytes_inp, encoding=enc, quotemarks=False, quote_newlines=quote_newlines), out2)
 
     def _test_quote_output_all(self, enc):
@@ -327 +380 @@
 
         # optional single quotes
-        check("foo",  "'foo'",  True)
-        check("\\",   "'\\'",   True)
-        check("$\"`", "'$\"`'", True)
-        check("\n",   "'\n'",   True, quote_newlines=False)
+        check(b"foo",  b"'foo'",  True)
+        check(b"\\",   b"'\\'",   True)
+        check(b"$\"`", b"'$\"`'", True)
+        check(b"\n",   b"'\n'",   True, quote_newlines=False)
 
         # mandatory single quotes
-        check("\"",   "'\"'")
+        check(b"\"",   b"'\"'")
 
         # double quotes
-        check("'",    "\"'\"")
-        check("\n",   "\"\\x0a\"", quote_newlines=True)
-        check("\x00", "\"\\x00\"")
+        check(b"'",    b"\"'\"")
+        check(b"\n",   b"\"\\x0a\"", quote_newlines=True)
+        check(b"\x00", b"\"\\x00\"")
 
         # invalid Unicode and astral planes
-        check(u"\uFDD0\uFDEF",       "\"\\ufdd0\\ufdef\"")
-        check(u"\uDC00\uD800",       "\"\\udc00\\ud800\"")
-        check(u"\uDC00\uD800\uDC00", "\"\\udc00\\U00010000\"")
-        check(u"\uD800\uDC00",       "\"\\U00010000\"")
-        check(u"\uD800\uDC01",       "\"\\U00010001\"")
-        check(u"\uD801\uDC00",       "\"\\U00010400\"")
-        check(u"\uDBFF\uDFFF",       "\"\\U0010ffff\"")
-        check(u"'\uDBFF\uDFFF",      "\"'\\U0010ffff\"")
-        check(u"\"\uDBFF\uDFFF",     "\"\\\"\\U0010ffff\"")
+        check(u"\uFDD0\uFDEF",       b"\"\\ufdd0\\ufdef\"")
+        check(u"\uDC00\uD800",       b"\"\\udc00\\ud800\"")
+        check(u"\uDC00\uD800\uDC00", b"\"\\udc00\\U00010000\"")
+        check(u"\uD800\uDC00",       b"\"\\U00010000\"")
+        check(u"\uD800\uDC01",       b"\"\\U00010001\"")
+        check(u"\uD801\uDC00",       b"\"\\U00010400\"")
+        check(u"\uDBFF\uDFFF",       b"\"\\U0010ffff\"")
+        check(u"'\uDBFF\uDFFF",      b"\"'\\U0010ffff\"")
+        check(u"\"\uDBFF\uDFFF",     b"\"\\\"\\U0010ffff\"")
 
         # invalid UTF-8
-        check("\xFF",                "b\"\\xff\"")
-        check("\x00\"$\\`\x80\xFF",  "b\"\\x00\\\"\\$\\\\\\`\\x80\\xff\"")
+        check(b"\xFF",                b"b\"\\xff\"")
+        check(b"\x00\"$\\`\x80\xFF",  b"b\"\\x00\\\"\\$\\\\\\`\\x80\\xff\"")
 
     def test_quote_output_ascii(self, enc='ascii'):
@@ -360 +413 @@
 
         self._test_quote_output_all(enc)
-        check(u"\u00D7",   "\"\\xd7\"")
-        check(u"'\u00D7",  "\"'\\xd7\"")
-        check(u"\"\u00D7", "\"\\\"\\xd7\"")
-        check(u"\u2621",   "\"\\u2621\"")
-        check(u"'\u2621",  "\"'\\u2621\"")
-        check(u"\"\u2621", "\"\\\"\\u2621\"")
-        check(u"\n",       "'\n'",      True, quote_newlines=False)
-        check(u"\n",       "\"\\x0a\"", quote_newlines=True)
+        check(u"\u00D7",   b"\"\\xd7\"")
+        check(u"'\u00D7",  b"\"'\\xd7\"")
+        check(u"\"\u00D7", b"\"\\\"\\xd7\"")
+        check(u"\u2621",   b"\"\\u2621\"")
+        check(u"'\u2621",  b"\"'\\u2621\"")
+        check(u"\"\u2621", b"\"\\\"\\u2621\"")
+        check(u"\n",       b"'\n'",      True, quote_newlines=False)
+        check(u"\n",       b"\"\\x0a\"", quote_newlines=True)
 
     def test_quote_output_latin1(self, enc='latin1'):
@@ -412 +465 @@
 class QuotePaths(ReallyEqualMixin, unittest.TestCase):
     def test_quote_path(self):
-        self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'")
-        self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'")
-        self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), "foo/bar")
-        self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), '"foo/\\x0abar"')
-        self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"')
-        self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), '"foo/\\x0abar"')
+        self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), b"'foo/bar'")
+        self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), b"'foo/bar'")
+        self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), b"foo/bar")
+        self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), b'"foo/\\x0abar"')
+        self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), b'"foo/\\x0abar"')
+        self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), b'"foo/\\x0abar"')
 
         self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"),
-                                   win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
+                                   win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'"))
         self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
-                                   win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
+                                   win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'"))
         self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
-                                   win32_other("C:\\foo", "\\\\?\\C:\\foo"))
+                                   win32_other(b"C:\\foo", b"\\\\?\\C:\\foo"))
         self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
-                                   win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
+                                   win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'"))
         self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
-                                   win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
+                                   win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'"))
         self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False),
-                                   win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"))
+                                   win32_other(b"\\\\foo\\bar", b"\\\\?\\UNC\\foo\\bar"))
 
     def test_quote_filepath(self):
         foo_bar_fp = FilePath(win32_other(u'C:\\foo\\bar', u'/foo/bar'))
         self.failUnlessReallyEqual(quote_filepath(foo_bar_fp),
-                                   win32_other("'C:\\foo\\bar'", "'/foo/bar'"))
+                                   win32_other(b"'C:\\foo\\bar'", b"'/foo/bar'"))
         self.failUnlessReallyEqual(quote_filepath(foo_bar_fp, quotemarks=True),
-                                   win32_other("'C:\\foo\\bar'", "'/foo/bar'"))
+                                   win32_other(b"'C:\\foo\\bar'", b"'/foo/bar'"))
         self.failUnlessReallyEqual(quote_filepath(foo_bar_fp, quotemarks=False),
-                                   win32_other("C:\\foo\\bar", "/foo/bar"))
+                                   win32_other(b"C:\\foo\\bar", b"/foo/bar"))
 
         if sys.platform == "win32":
             foo_longfp = FilePath(u'\\\\?\\C:\\foo')
             self.failUnlessReallyEqual(quote_filepath(foo_longfp),
-                                       "'C:\\foo'")
+                                       b"'C:\\foo'")
             self.failUnlessReallyEqual(quote_filepath(foo_longfp, quotemarks=True),
-                                       "'C:\\foo'")
+                                       b"'C:\\foo'")
             self.failUnlessReallyEqual(quote_filepath(foo_longfp, quotemarks=False),
-                                       "C:\\foo")
+                                       b"C:\\foo")
 
 
@@ -502 +555 @@
 class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
-    argv = 'lumi\xc3\xa8re'
+    argv = b'lumi\xc3\xa8re'
     platform = 'linux2'
     filesystem_encoding = 'UTF-8'
     io_encoding = 'UTF-8'
-    dirlist = ['test_file', '\xc3\x84rtonwall.mp3', 'Blah blah.txt']
+    dirlist = [b'test_file', b'\xc3\x84rtonwall.mp3', b'Blah blah.txt']
 
 class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
-    argv = 'lumi\xe8re'
+    argv = b'lumi\xe8re'
     platform = 'linux2'
     filesystem_encoding = 'ISO-8859-1'
     io_encoding = 'ISO-8859-1'
-    dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3']
+    dirlist = [b'test_file', b'Blah blah.txt', b'\xc4rtonwall.mp3']
 
 class Windows(EncodingUtil, unittest.TestCase):
     uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
-    argv = 'lumi\xc3\xa8re'
+    argv = b'lumi\xc3\xa8re'
     platform = 'win32'
     filesystem_encoding = 'mbcs'
@@ -526 +579 @@
 class MacOSXLeopard(EncodingUtil, unittest.TestCase):
     uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
-    output = 'lumi\xc3\xa8re'
+    output = b'lumi\xc3\xa8re'
     platform = 'darwin'
     filesystem_encoding = 'utf-8'
@@ -549 +602 @@
 class TestToFromStr(ReallyEqualMixin, unittest.TestCase):
     def test_to_str(self):
-        self.failUnlessReallyEqual(to_str("foo"), "foo")
-        self.failUnlessReallyEqual(to_str("lumi\xc3\xa8re"), "lumi\xc3\xa8re")
-        self.failUnlessReallyEqual(to_str("\xFF"), "\xFF")  # passes through invalid UTF-8 -- is this what we want?
-        self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), "lumi\xc3\xa8re")
+        self.failUnlessReallyEqual(to_str(b"foo"), b"foo")
+        self.failUnlessReallyEqual(to_str(b"lumi\xc3\xa8re"), b"lumi\xc3\xa8re")
+        self.failUnlessReallyEqual(to_str(b"\xFF"), b"\xFF")  # passes through invalid UTF-8 -- is this what we want?
+        self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), b"lumi\xc3\xa8re")
         self.failUnlessReallyEqual(to_str(None), None)
 
     def test_from_utf8_or_none(self):
         self.failUnlessRaises(AssertionError, from_utf8_or_none, u"foo")
-        self.failUnlessReallyEqual(from_utf8_or_none("lumi\xc3\xa8re"), u"lumi\u00E8re")
+        self.failUnlessReallyEqual(from_utf8_or_none(b"lumi\xc3\xa8re"), u"lumi\u00E8re")
         self.failUnlessReallyEqual(from_utf8_or_none(None), None)
-        self.failUnlessRaises(UnicodeDecodeError, from_utf8_or_none, "\xFF")
+        self.failUnlessRaises(UnicodeDecodeError, from_utf8_or_none, b"\xFF")

src/allmydata/test/test_happiness.py ¶

@@ -1 +1 @@
 # -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # We omit dict, just in case newdict breaks things.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
 
 from twisted.trial import unittest

src/allmydata/test/test_util.py ¶

@@ +1 @@
+"""
+Ported to Python3.
+"""
+
 from __future__ import print_function
-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 import six
 import os, time, sys
@@ -20 +30 @@
 class IDLib(unittest.TestCase):
     def test_nodeid_b2a(self):
-        self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32)
+        self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32)
 
 
@@ -86 +96 @@
         fileutil.make_dirs(basedir)
         fn = os.path.join(basedir, "here")
-        fileutil.write_atomically(fn, "one")
-        self.failUnlessEqual(fileutil.read(fn), "one")
-        fileutil.write_atomically(fn, "two", mode="") # non-binary
-        self.failUnlessEqual(fileutil.read(fn), "two")
+        fileutil.write_atomically(fn, b"one", "b")
+        self.failUnlessEqual(fileutil.read(fn), b"one")
+        fileutil.write_atomically(fn, u"two", mode="") # non-binary
+        self.failUnlessEqual(fileutil.read(fn), b"two")
 
     def test_rename(self):
@@ -112 +122 @@
 
         # when only dest exists
-        fileutil.write(dest_path,   "dest")
+        fileutil.write(dest_path,   b"dest")
         self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
-        self.failUnlessEqual(fileutil.read(dest_path),   "dest")
+        self.failUnlessEqual(fileutil.read(dest_path),   b"dest")
 
         # when both exist
-        fileutil.write(source_path, "source")
+        fileutil.write(source_path, b"source")
         self.failUnlessRaises(OSError, fileutil.rename_no_overwrite, source_path, dest_path)
-        self.failUnlessEqual(fileutil.read(source_path), "source")
-        self.failUnlessEqual(fileutil.read(dest_path),   "dest")
+        self.failUnlessEqual(fileutil.read(source_path), b"source")
+        self.failUnlessEqual(fileutil.read(dest_path),   b"dest")
 
         # when only source exists
         os.remove(dest_path)
         fileutil.rename_no_overwrite(source_path, dest_path)
-        self.failUnlessEqual(fileutil.read(dest_path), "source")
+        self.failUnlessEqual(fileutil.read(dest_path), b"source")
         self.failIf(os.path.exists(source_path))
 
@@ -139 +149 @@
 
         # when only replaced exists
-        fileutil.write(replaced_path,    "foo")
+        fileutil.write(replaced_path,   b"foo")
         self.failUnlessRaises(fileutil.ConflictError, fileutil.replace_file, replaced_path, replacement_path)
-        self.failUnlessEqual(fileutil.read(replaced_path), "foo")
+        self.failUnlessEqual(fileutil.read(replaced_path), b"foo")
 
         # when both replaced and replacement exist
-        fileutil.write(replacement_path, "bar")
+        fileutil.write(replacement_path, b"bar")
         fileutil.replace_file(replaced_path, replacement_path)
-        self.failUnlessEqual(fileutil.read(replaced_path), "bar")
+        self.failUnlessEqual(fileutil.read(replaced_path), b"bar")
         self.failIf(os.path.exists(replacement_path))
 
         # when only replacement exists
         os.remove(replaced_path)
-        fileutil.write(replacement_path, "bar")
+        fileutil.write(replacement_path, b"bar")
         fileutil.replace_file(replaced_path, replacement_path)
-        self.failUnlessEqual(fileutil.read(replaced_path), "bar")
+        self.failUnlessEqual(fileutil.read(replaced_path), b"bar")
         self.failIf(os.path.exists(replacement_path))
 
@@ -171 +181 @@
 
     def test_abspath_expanduser_unicode(self):
-        self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring")
-
-        saved_cwd = os.path.normpath(os.getcwdu())
+        self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring")
+
+        saved_cwd = os.path.normpath(os.getcwd())
+        if PY2:
+            saved_cwd = saved_cwd.decode("utf8")
         abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
         abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False)
-        self.failUnless(isinstance(saved_cwd, unicode), saved_cwd)
-        self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd)
+        self.failUnless(isinstance(saved_cwd, str), saved_cwd)
+        self.failUnless(isinstance(abspath_cwd, str), abspath_cwd)
         if sys.platform == "win32":
             self.failUnlessReallyEqual(abspath_cwd, fileutil.to_windows_long_path(saved_cwd))
@@ -238 +250 @@
                 for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'):
                     uabspath = fileutil.abspath_expanduser_unicode(upath)
-                    self.failUnless(isinstance(uabspath, unicode), uabspath)
+                    self.failUnless(isinstance(uabspath, str), uabspath)
 
                     uabspath_notlong = fileutil.abspath_expanduser_unicode(upath, long_path=False)
-                    self.failUnless(isinstance(uabspath_notlong, unicode), uabspath_notlong)
+                    self.failUnless(isinstance(uabspath_notlong, str), uabspath_notlong)
             finally:
                 os.chdir(saved_cwd)
@@ -294 +306 @@
         self.addCleanup(_cleanup)
 
-        fileutil.write(long_path, "test")
+        fileutil.write(long_path, b"test")
         self.failUnless(os.path.exists(long_path))
-        self.failUnlessEqual(fileutil.read(long_path), "test")
+        self.failUnlessEqual(fileutil.read(long_path), b"test")
         _cleanup()
         self.failIf(os.path.exists(long_path))
@@ -354 +366 @@
         # create a file
         f = os.path.join(basedir, "1.txt")
-        fileutil.write(f, "a"*10)
+        fileutil.write(f, b"a"*10)
         fileinfo = fileutil.get_pathinfo(f)
         self.failUnlessTrue(fileinfo.isfile)
@@ -382 +394 @@
 
         f = os.path.join(basedir, "1.txt")
-        fileutil.write(f, "a"*10)
+        fileutil.write(f, b"a"*10)
 
         # create a symlink pointing to 1.txt
@@ -395 +407 @@
     def test_encrypted_tempfile(self):
         f = EncryptedTemporaryFile()
-        f.write("foobar")
+        f.write(b"foobar")
         f.close()
 
@@ -410 +422 @@
     def test_PollMixin_False_then_True(self):
         i = iter([False, True])
-        d = self.pm.poll(check_f=i.next,
+        d = self.pm.poll(check_f=lambda: next(i),
                          pollinterval=0.1)
         return d
@@ -455 +467 @@
         data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])
         back = yamlutil.safe_load(data)
-        self.failUnlessEqual(type(back[0]), unicode)
-        self.failUnlessEqual(type(back[1]), unicode)
-        self.failUnlessEqual(type(back[2]), unicode)
+        self.assertIsInstance(back[0], str)
+        self.assertIsInstance(back[1], str)
+        self.assertIsInstance(back[2], str)

src/allmydata/util/_python3.py ¶

@@ -23 +23 @@
     "allmydata.crypto.util",
     "allmydata.hashtree",
+    "allmydata.immutable.happiness_upload",
     "allmydata.test.common_py3",
     "allmydata.util._python3",
@@ -32 +33 @@
     "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
+    "allmydata.util.fileutil",
     "allmydata.util.dictutil",
+    "allmydata.util.encodingutil",
     "allmydata.util.gcutil",
     "allmydata.util.hashutil",
@@ -58 +61 @@
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",
+    "allmydata.test.test_encodingutil",
+    "allmydata.test.test_happiness",
     "allmydata.test.test_hashtree",
     "allmydata.test.test_hashutil",
@@ -70 +75 @@
     "allmydata.test.test_statistics",
     "allmydata.test.test_time_format",
+    "allmydata.test.test_util",
     "allmydata.test.test_version",
 ]
-
 
 if __name__ == '__main__':

src/allmydata/util/encodingutil.py ¶

@@ -2 +2 @@
 Functions used to convert inputs from whatever encoding used in the system to
 unicode and back.
+
+Ported to Python 3.
+
+Once Python 2 support is dropped, most of this module will obsolete, since
+Unicode is the default everywhere in Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, PY3, native_str
+if PY2:
+    # We omit str() because that seems too tricky to get right.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+
+from past.builtins import unicode
 
 import sys, os, re, locale
+import unicodedata
 
 from allmydata.util.assertutil import precondition, _assert
@@ -63 +80 @@
     check_encoding(io_encoding)
 
-    is_unicode_platform = sys.platform in ["win32", "darwin"]
+    is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"]
 
     # Despite the Unicode-mode FilePath support added to Twisted in
     # <https://twistedmatrix.com/trac/ticket/7805>, we can't yet use
     # Unicode-mode FilePaths with INotify on non-Windows platforms
-    # due to <https://twistedmatrix.com/trac/ticket/7928>.
-    use_unicode_filepath = sys.platform == "win32"
+    # due to <https://twistedmatrix.com/trac/ticket/7928>. Supposedly
+    # 7928 is fixed, though...
+    use_unicode_filepath = PY3 or sys.platform == "win32"
 
 _reload()
@@ -90 +108 @@
     Decode given argv element to unicode. If this fails, raise a UsageError.
     """
-    precondition(isinstance(s, str), s)
+    if isinstance(s, unicode):
+        return s
+
+    precondition(isinstance(s, bytes), s)
 
     try:
@@ -115 +136 @@
     should be true; on Windows, this uses a mangled encoding that will be reversed by
     code in runner.py.
+
+    On Python 3, just return the string unchanged, since argv is unicode.
     """
     precondition(isinstance(s, unicode), s)
+    if PY3:
+        return s
 
     if mangle and sys.platform == "win32":
         # This must be the same as 'mangle' in bin/tahoe-script.template.
-        return str(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s))
+        return bytes(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s), io_encoding)
     else:
         return s.encode(io_encoding)
@@ -126 +151 @@
 def unicode_to_url(s):
     """
-    Encode an unicode object used in an URL.
+    Encode an unicode object used in an URL to bytes.
     """
     # According to RFC 2718, non-ascii characters in URLs must be UTF-8 encoded.
@@ -135 +160 @@
     #return s.encode('utf-8')
 
-def to_str(s):
-    if s is None or isinstance(s, str):
+def to_str(s):  # TODO rename to to_bytes
+    if s is None or isinstance(s, bytes):
         return s
     return s.encode('utf-8')
 
 def from_utf8_or_none(s):
-    precondition(isinstance(s, (NoneType, str)), s)
+    precondition(isinstance(s, bytes) or s is None, s)
     if s is None:
         return s
     return s.decode('utf-8')
 
-PRINTABLE_ASCII = re.compile(r'^[\n\r\x20-\x7E]*$',          re.DOTALL)
-PRINTABLE_8BIT  = re.compile(r'^[\n\r\x20-\x7E\x80-\xFF]*$', re.DOTALL)
+PRINTABLE_ASCII = re.compile(br'^[\n\r\x20-\x7E]*$',          re.DOTALL)
+PRINTABLE_8BIT  = re.compile(br'^[\n\r\x20-\x7E\x80-\xFF]*$', re.DOTALL)
 
 def is_printable_ascii(s):
@@ -161 +186 @@
         out = s.encode(io_encoding)
     except (UnicodeEncodeError, UnicodeDecodeError):
-        raise UnicodeEncodeError(io_encoding, s, 0, 0,
-                                 "A string could not be encoded as %s for output to the terminal:\n%r" %
-                                 (io_encoding, repr(s)))
+        raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0,
+                                 native_str("A string could not be encoded as %s for output to the terminal:\n%r" %
+                                 (io_encoding, repr(s))))
 
     if PRINTABLE_8BIT.search(out) is None:
-        raise UnicodeEncodeError(io_encoding, s, 0, 0,
-                                 "A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" %
-                                 (io_encoding, repr(s)))
+        raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0,
+                                 native_str("A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" %
+                                 (io_encoding, repr(s))))
     return out
 
@@ -189 +214 @@
         return u'\\x%02x' % (codepoint,)
 
-def _str_escape(m, quote_newlines):
+def _str_escape(m, quote_newlines):  # TODO rename to _bytes_escape
+    """
+    Takes a re match on bytes, the result is escaped bytes of group(0).
+    """
     c = m.group(0)
-    if c == '"' or c == '$' or c == '`' or c == '\\':
-        return '\\' + c
-    elif c == '\n' and not quote_newlines:
+    if c == b'"' or c == b'$' or c == b'`' or c == b'\\':
+        return b'\\' + c
+    elif c == b'\n' and not quote_newlines:
         return c
     else:
-        return '\\x%02x' % (ord(c),)
+        return b'\\x%02x' % (ord(c),)
 
 MUST_DOUBLE_QUOTE_NL = re.compile(u'[^\\x20-\\x26\\x28-\\x7E\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFC]', re.DOTALL)
@@ -206 +234 @@
                                re.DOTALL)
 
-ESCAPABLE_8BIT    = re.compile( r'[^ !#\x25-\x5B\x5D-\x5F\x61-\x7E]', re.DOTALL)
+ESCAPABLE_8BIT    = re.compile( br'[^ !#\x25-\x5B\x5D-\x5F\x61-\x7E]', re.DOTALL)
 
 def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None):
@@ -222 +250 @@
     If not explicitly given, quote_newlines is True when quotemarks is True.
     """
-    precondition(isinstance(s, (str, unicode)), s)
+    precondition(isinstance(s, (bytes, unicode)), s)
     if quote_newlines is None:
         quote_newlines = quotemarks
 
-    if isinstance(s, str):
+    if isinstance(s, bytes):
         try:
             s = s.decode('utf-8')
         except UnicodeDecodeError:
-            return 'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _str_escape(m, quote_newlines), s),)
+            return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _str_escape(m, quote_newlines), s),)
 
     must_double_quote = quote_newlines and MUST_DOUBLE_QUOTE_NL or MUST_DOUBLE_QUOTE
@@ -236 +264 @@
         try:
             out = s.encode(encoding or io_encoding)
-            if quotemarks or out.startswith('"'):
-                return "'%s'" % (out,)
+            if quotemarks or out.startswith(b'"'):
+                return b"'%s'" % (out,)
             else:
                 return out
@@ -244 +272 @@
 
     escaped = ESCAPABLE_UNICODE.sub(lambda m: _unicode_escape(m, quote_newlines), s)
-    return '"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),)
+    return b'"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),)
 
 def quote_path(path, quotemarks=True):
-    return quote_output("/".join(map(to_str, path)), quotemarks=quotemarks, quote_newlines=True)
+    return quote_output(b"/".join(map(to_str, path)), quotemarks=quotemarks, quote_newlines=True)
 
 def quote_local_unicode_path(path, quotemarks=True):
@@ -276 +304 @@
 
 def to_filepath(path):
-    precondition(isinstance(path, unicode if use_unicode_filepath else basestring),
+    precondition(isinstance(path, unicode if use_unicode_filepath else (bytes, unicode)),
                  path=path)
 
@@ -291 +319 @@
 
 def _decode(s):
-    precondition(isinstance(s, basestring), s=s)
+    precondition(isinstance(s, (bytes, unicode)), s=s)
 
     if isinstance(s, bytes):
@@ -357 +385 @@
 def listdir_filepath(fp):
     return listdir_unicode(unicode_from_filepath(fp))
+
+
+# 'x' at the end of a variable name indicates that it holds a Unicode string that may not
+# be NFC-normalized.
+def normalize(namex):
+    return unicodedata.normalize('NFC', namex)

src/allmydata/util/fileutil.py ¶

@@ -1 @@
-from __future__ import print_function
-
 """
+Ported to Python3.
+
 Futz with files like a pro.
 """
+
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # open is not here because we want to use native strings on Py2
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import sys, os, stat, tempfile, time, binascii
@@ -254 +264 @@
 
 def write_atomically(target, contents, mode="b"):
+    assert (
+        isinstance(contents, bytes) and "b" in mode or
+        isinstance(contents, str) and "t" in mode or mode == ""), (type(contents), mode)
     with open(target+".tmp", "w"+mode) as f:
         f.write(contents)
@@ -278 +291 @@
 
 def precondition_abspath(path):
-    if not isinstance(path, unicode):
+    if not isinstance(path, str):
         raise AssertionError("an abspath must be a Unicode string")
 
@@ -310 +323 @@
     On Windows, the result will be a long path unless long_path is given as False.
     """
-    if not isinstance(path, unicode):
+    if not isinstance(path, str):
         raise AssertionError("paths must be Unicode strings")
     if base is not None and long_path:
@@ -331 +344 @@
     if not os.path.isabs(path):
         if base is None:
-            path = os.path.join(os.getcwdu(), path)
+            cwd = os.getcwd()
+            if PY2:
+                cwd = cwd.decode('utf8')
+            path = os.path.join(cwd, path)
         else:
             path = os.path.join(base, path)
@@ -416 +432 @@
     # Based on <http://stackoverflow.com/questions/2608200/problems-with-umlauts-in-python-appdata-environvent-variable/2608368#2608368>,
     # with improved error handling. Returns None if there is no enivronment variable of the given name.
-    if not isinstance(name, unicode):
+    if not isinstance(name, str):
         raise AssertionError("name must be Unicode")
 

src/allmydata/util/statistics.py ¶

@@ -21 +21 @@
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from functools import reduce
-
 from allmydata.util.mathutil import round_sigfigs
 import math
+from functools import reduce
 import sys
 

tox.ini ¶

@@ -51 +51 @@
 [testenv:py36]
 # On macOS, git inside of ratchet.sh needs $HOME.
-passenv = HOME
+passenv = {[testenv]passenv} HOME
 commands = {toxinidir}/misc/python3/ratchet.sh