Changeset 5e6201ae in trunk

Timestamp:

2020-09-28T15:35:15Z (5 years ago)

Author:

GitHub <noreply@…>

Branches:

master

Children:

587d9ef, cc7a3bc

Parents:

4e7e847 (diff), d7b5230f (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Itamar Turner-Trauring <itamar@…> (2020-09-28 15:35:15)

git-committer:

GitHub <noreply@…> (2020-09-28 15:35:15)

Message:

Merge pull request #833 from tahoe-lafs/3446.test-helper-python-3

Port test_helper.py to Python 3

Fixes ticket:3446

Files:

• newsfragments/3446.minor (added)
• src/allmydata/immutable/offloaded.py (modified) (2 diffs)
• src/allmydata/immutable/upload.py (modified) (2 diffs)
• src/allmydata/interfaces.py (modified) (14 diffs)
• src/allmydata/test/test_helper.py (modified) (11 diffs)
• src/allmydata/util/_python3.py (modified) (1 diff)

src/allmydata/immutable/offloaded.py ¶

@@ -204 +204 @@
         assert interfaces.IUploadResults.providedBy(ur), ur
         vcapstr = ur.get_verifycapstr()
-        precondition(isinstance(vcapstr, str), vcapstr)
+        precondition(isinstance(vcapstr, bytes), vcapstr)
         v = uri.from_string(vcapstr)
         f_times = self._fetcher.get_times()
@@ -493 +493 @@
 
     name = "helper"
-    VERSION = { "http://allmydata.org/tahoe/protocols/helper/v1" :
+    VERSION = { b"http://allmydata.org/tahoe/protocols/helper/v1" :
                  { },
-                "application-version": str(allmydata.__full_version__),
+                b"application-version": allmydata.__full_version__.encode("utf-8"),
                 }
     MAX_UPLOAD_STATUSES = 10

src/allmydata/immutable/upload.py ¶

@@ -1817 +1817 @@
     def _got_helper(self, helper):
         self.log("got helper connection, getting versions")
-        default = { "http://allmydata.org/tahoe/protocols/helper/v1" :
+        default = { b"http://allmydata.org/tahoe/protocols/helper/v1" :
                     { },
-                    "application-version": b"unknown: no get_version()",
+                    b"application-version": b"unknown: no get_version()",
                     }
         d = add_version_to_remote_reference(helper, default)
@@ -1825 +1825 @@
 
     def _got_versioned_helper(self, helper):
-        needed = "http://allmydata.org/tahoe/protocols/helper/v1"
+        needed = b"http://allmydata.org/tahoe/protocols/helper/v1"
         if needed not in helper.version:
             raise InsufficientVersionError(needed, helper.version)

src/allmydata/interfaces.py ¶

@@ -3 +3 @@
 
 Ported to Python 3.
+
+Note that for RemoteInterfaces, the __remote_name__ needs to be a native string because of https://github.com/warner/foolscap/blob/43f4485a42c9c28e2c79d655b3a9e24d4e6360ca/src/foolscap/remoteinterface.py#L67
 """
 from __future__ import absolute_import
@@ -9 +11 @@
 from __future__ import unicode_literals
 
-from future.utils import PY2
+from future.utils import PY2, native_str
 if PY2:
     # Don't import object/str/dict/etc. types, so we don't break any
@@ -106 +108 @@
 
 class RIStorageServer(RemoteInterface):
-    __remote_name__ = b"RIStorageServer.tahoe.allmydata.com"
+    __remote_name__ = native_str("RIStorageServer.tahoe.allmydata.com")
 
     def get_version():
@@ -2837 +2839 @@
     # debug stuff
 
-    def upload_random_data_from_file(size=int, convergence=str):
+    def upload_random_data_from_file(size=int, convergence=bytes):
         return str
 
-    def download_to_tempfile_and_delete(uri=str):
+    def download_to_tempfile_and_delete(uri=bytes):
         return None
 
@@ -2847 +2849 @@
         keys are 'VmPeak', 'VmSize', and 'VmData'. The values are integers,
         measuring memory consupmtion in bytes."""
-        return DictOf(str, int)
+        return DictOf(bytes, int)
 
     def speed_test(count=int, size=int, mutable=Any()):
@@ -2872 +2874 @@
 
 
-UploadResults = Any() #DictOf(str, str)
+UploadResults = Any() #DictOf(bytes, bytes)
 
 
 class RIEncryptedUploadable(RemoteInterface):
-    __remote_name__ = b"RIEncryptedUploadable.tahoe.allmydata.com"
+    __remote_name__ = native_str("RIEncryptedUploadable.tahoe.allmydata.com")
 
     def get_size():
@@ -2885 +2887 @@
 
     def read_encrypted(offset=Offset, length=ReadSize):
-        return ListOf(str)
+        return ListOf(bytes)
 
     def close():
@@ -2892 +2894 @@
 
 class RICHKUploadHelper(RemoteInterface):
-    __remote_name__ = b"RIUploadHelper.tahoe.allmydata.com"
+    __remote_name__ = native_str("RIUploadHelper.tahoe.allmydata.com")
 
     def get_version():
@@ -2898 +2900 @@
         Return a dictionary of version information.
         """
-        return DictOf(str, Any())
+        return DictOf(bytes, Any())
 
     def upload(reader=RIEncryptedUploadable):
@@ -2905 +2907 @@
 
 class RIHelper(RemoteInterface):
-    __remote_name__ = b"RIHelper.tahoe.allmydata.com"
+    __remote_name__ = native_str("RIHelper.tahoe.allmydata.com")
 
     def get_version():
@@ -2911 +2913 @@
         Return a dictionary of version information.
         """
-        return DictOf(str, Any())
+        return DictOf(bytes, Any())
 
     def upload_chk(si=StorageIndex):
@@ -2932 +2934 @@
 
 class RIStatsProvider(RemoteInterface):
-    __remote_name__ = b"RIStatsProvider.tahoe.allmydata.com"
+    __remote_name__ = native_str("RIStatsProvider.tahoe.allmydata.com")
     """
     Provides access to statistics and monitoring information.
@@ -2945 +2947 @@
         internally)
         """
-        return DictOf(str, DictOf(str, ChoiceOf(float, int, long, None)))
+        return DictOf(bytes, DictOf(bytes, ChoiceOf(float, int, long, None)))
 
 
 class RIStatsGatherer(RemoteInterface):
-    __remote_name__ = b"RIStatsGatherer.tahoe.allmydata.com"
+    __remote_name__ = native_str("RIStatsGatherer.tahoe.allmydata.com")
     """
     Provides a monitoring service for centralised collection of stats
     """
 
-    def provide(provider=RIStatsProvider, nickname=str):
+    def provide(provider=RIStatsProvider, nickname=bytes):
         """
         @param provider: a stats collector instance that should be polled
@@ -2966 +2968 @@
     def get_stats():
         """
-        returns a dictionary, with str keys representing the names of stats
+        returns a dictionary, with bytes keys representing the names of stats
         to be monitored, and numeric values.
         """

src/allmydata/test/test_helper.py ¶

@@ +1 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os
 from twisted.internet import defer
@@ -19 +28 @@
 MiB = 1024*1024
 
-DATA = "I need help\n" * 1000
+DATA = b"I need help\n" * 1000
 
 class CHKUploadHelper_fake(offloaded.CHKUploadHelper):
@@ -34 +43 @@
                             "size": size,
                             }
-                ueb_hash = "fake"
-                v = uri.CHKFileVerifierURI(self._storage_index, "x"*32,
+                ueb_hash = b"fake"
+                v = uri.CHKFileVerifierURI(self._storage_index, b"x"*32,
                                            needed_shares, total_shares, size)
                 _UR = upload.UploadResults
@@ -57 +66 @@
 class Helper_fake_upload(offloaded.Helper):
     def _make_chk_upload_helper(self, storage_index, lp):
-        si_s = si_b2a(storage_index)
+        si_s = str(si_b2a(storage_index), "utf-8")
         incoming_file = os.path.join(self._chk_incoming, si_s)
         encoding_file = os.path.join(self._chk_encoding, si_s)
@@ -70 +79 @@
     def _check_chk(self, storage_index, lp):
         res = upload.HelperUploadResults()
-        res.uri_extension_hash = hashutil.uri_extension_hash("")
+        res.uri_extension_hash = hashutil.uri_extension_hash(b"")
 
         # we're pretending that the file they're trying to upload was already
@@ -128 +137 @@
             EMPTY_CLIENT_CONFIG,
         )
-        self.s.secret_holder = client.SecretHolder("lease secret", "converge")
+        self.s.secret_holder = client.SecretHolder(b"lease secret", b"converge")
         self.s.startService()
 
@@ -135 +144 @@
         # we never actually use this for network traffic, so it can use a
         # bogus host/port
-        t.setLocation("bogus:1234")
+        t.setLocation(b"bogus:1234")
 
     def setUpHelper(self, basedir, helper_class=Helper_fake_upload):
@@ -163 +172 @@
             assert u._helper
 
-            return upload_data(u, DATA, convergence="some convergence string")
+            return upload_data(u, DATA, convergence=b"some convergence string")
         d.addCallback(_ready)
         def _uploaded(results):
             the_uri = results.get_uri()
-            assert "CHK" in the_uri
+            assert b"CHK" in the_uri
         d.addCallback(_uploaded)
 
@@ -196 +205 @@
         segsize = mathutil.next_multiple(segsize, k)
 
-        key = hashutil.convergence_hash(k, n, segsize, DATA, "test convergence string")
+        key = hashutil.convergence_hash(k, n, segsize, DATA, b"test convergence string")
         assert len(key) == 16
         encryptor = aes.create_encryptor(key)
         SI = hashutil.storage_index_hash(key)
-        SI_s = si_b2a(SI)
+        SI_s = str(si_b2a(SI), "utf-8")
         encfile = os.path.join(self.basedir, "CHK_encoding", SI_s)
         f = open(encfile, "wb")
@@ -213 +222 @@
         def _ready(res):
             assert u._helper
-            return upload_data(u, DATA, convergence="test convergence string")
+            return upload_data(u, DATA, convergence=b"test convergence string")
         d.addCallback(_ready)
         def _uploaded(results):
             the_uri = results.get_uri()
-            assert "CHK" in the_uri
+            assert b"CHK" in the_uri
         d.addCallback(_uploaded)
 
@@ -240 +249 @@
             assert u._helper
 
-            return upload_data(u, DATA, convergence="some convergence string")
+            return upload_data(u, DATA, convergence=b"some convergence string")
         d.addCallback(_ready)
         def _uploaded(results):
             the_uri = results.get_uri()
-            assert "CHK" in the_uri
+            assert b"CHK" in the_uri
         d.addCallback(_uploaded)
 

src/allmydata/util/_python3.py ¶

@@ -94 +94 @@
     "allmydata.test.test_hashtree",
     "allmydata.test.test_hashutil",
+    "allmydata.test.test_helper",
     "allmydata.test.test_humanreadable",
     "allmydata.test.test_immutable",