Changeset 89c2aac in trunk

Timestamp:

2021-11-16T22:08:31Z (4 years ago)

Author:

meejah <meejah@…>

Branches:

master

Children:

d4fc14f

Parents:

bf5e682

git-author:

meejah <meejah@…> (2021-10-25 21:52:01)

git-committer:

meejah <meejah@…> (2021-11-16 22:08:31)

Message:

working test of 'in the wild' data, working converters

Location:

src/allmydata

Files:

• storage/crawler.py (modified) (4 diffs)
• test/test_storage_web.py (modified) (1 diff)

src/allmydata/storage/crawler.py ¶

@@ -35 +35 @@
     """
     # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list
-    # ["leases-per-share-histogram"] gets str keys instead of int
+    # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int
     # ["cycle-start-finish-times"] from 2-tuple to list
     # ["configured-expiration-mode"] from 4-tuple to list
@@ -43 +43 @@
             "Unknown version {version} in pickle state".format(**state)
         )
-
-    def convert_lpsh(value):
-        return {
-            str(k): v
-            for k, v in value.items()
-        }
 
     def convert_cem(value):
@@ -60 +54 @@
         ]
 
-    def convert_history(value):
-        print("convert history")
-        print(value)
+    def convert_ctd(value):
+        ctd_converter = {
+            "lease-age-histogram": lambda value: {
+                "{},{}".format(k[0], k[1]): v
+                for k, v in value.items()
+            },
+            "corrupt-shares": lambda value: [
+                list(x)
+                for x in value
+            ],
+        }
         return {
-            str(k): v
-            for k, v in value
+            k: ctd_converter.get(k, lambda z: z)(v)
+            for k, v in value.items()
         }
 
+    # we don't convert "history" here because that's in a separate
+    # file; see expirer.py
     converters = {
-        "cycle-to-date": list,
-        "leases-per-share-histogram": convert_lpsh,
+        "cycle-to-date": convert_ctd,
         "cycle-starte-finish-times": list,
         "configured-expiration-mode": convert_cem,
-        "history": convert_history,
     }
 
@@ -117 +119 @@
     with state_path.open("r") as f:
         state = pickle.load(f)
-    state = convert_pickle(state)
+    new_state = convert_pickle(state)
     json_state_path = state_path.siblingExtension(".json")
     with json_state_path.open("w") as f:
-        json.dump(state, f)
+        json.dump(new_state, f)
     # we've written the JSON, delete the pickle
     state_path.remove()

src/allmydata/test/test_storage_web.py ¶

@@ -1161 +1161 @@
         self.assertTrue(test_pickle.siblingExtension(".json").exists())
 
+        self.assertEqual(
+            serial.load(),
+            {
+                u'last-complete-prefix': None,
+                u'version': 1,
+                u'current-cycle-start-time': 1635003106.611748,
+                u'last-cycle-finished': 312,
+                u'cycle-to-date': {
+                    u'leases-per-share-histogram': {
+                        u'1': 36793,
+                        u'2': 1,
+                    },
+                    u'space-recovered': {
+                        u'examined-buckets-immutable': 17183,
+                        u'configured-buckets-mutable': 0,
+                        u'examined-shares-mutable': 1796,
+                        u'original-shares-mutable': 1563,
+                        u'configured-buckets-immutable': 0,
+                        u'original-shares-immutable': 27926,
+                        u'original-diskbytes-immutable': 431149056,
+                        u'examined-shares-immutable': 34998,
+                        u'original-buckets': 14661,
+                        u'actual-shares-immutable': 0,
+                        u'configured-shares': 0,
+                        u'original-buckets-mutable': 899,
+                        u'actual-diskbytes': 4096,
+                        u'actual-shares-mutable': 0,
+                        u'configured-buckets': 1,
+                        u'examined-buckets-unknown': 14,
+                        u'actual-sharebytes': 0,
+                        u'original-shares': 29489,
+                        u'actual-buckets-immutable': 0,
+                        u'original-sharebytes': 312664812,
+                        u'examined-sharebytes-immutable': 383801602,
+                        u'actual-shares': 0,
+                        u'actual-sharebytes-immutable': 0,
+                        u'original-diskbytes': 441643008,
+                        u'configured-diskbytes-mutable': 0,
+                        u'configured-sharebytes-immutable': 0,
+                        u'configured-shares-mutable': 0,
+                        u'actual-diskbytes-immutable': 0,
+                        u'configured-diskbytes-immutable': 0,
+                        u'original-diskbytes-mutable': 10489856,
+                        u'actual-sharebytes-mutable': 0,
+                        u'configured-sharebytes': 0,
+                        u'examined-shares': 36794,
+                        u'actual-diskbytes-mutable': 0,
+                        u'actual-buckets': 1,
+                        u'original-buckets-immutable': 13761,
+                        u'configured-sharebytes-mutable': 0,
+                        u'examined-sharebytes': 390369660,
+                        u'original-sharebytes-immutable': 308125753,
+                        u'original-sharebytes-mutable': 4539059,
+                        u'actual-buckets-mutable': 0,
+                        u'examined-buckets-mutable': 1043,
+                        u'configured-shares-immutable': 0,
+                        u'examined-diskbytes': 476598272,
+                        u'examined-diskbytes-mutable': 9154560,
+                        u'examined-sharebytes-mutable': 6568058,
+                        u'examined-buckets': 18241,
+                        u'configured-diskbytes': 4096,
+                        u'examined-diskbytes-immutable': 467443712},
+                    u'corrupt-shares': [
+                        [u'2dn6xnlnsqwtnapwxfdivpm3s4', 4],
+                        [u'2dn6xnlnsqwtnapwxfdivpm3s4', 1],
+                        [u'2rrzthwsrrxolevmwdvbdy3rqi', 4],
+                        [u'2rrzthwsrrxolevmwdvbdy3rqi', 1],
+                        [u'2skfngcto6h7eqmn4uo7ntk3ne', 4],
+                        [u'2skfngcto6h7eqmn4uo7ntk3ne', 1],
+                        [u'32d5swqpqx2mwix7xmqzvhdwje', 4],
+                        [u'32d5swqpqx2mwix7xmqzvhdwje', 1],
+                        [u'5mmayp66yflmpon3o6unsnbaca', 4],
+                        [u'5mmayp66yflmpon3o6unsnbaca', 1],
+                        [u'6ixhpvbtre7fnrl6pehlrlflc4', 4],
+                        [u'6ixhpvbtre7fnrl6pehlrlflc4', 1],
+                        [u'ewzhvswjsz4vp2bqkb6mi3bz2u', 4],
+                        [u'ewzhvswjsz4vp2bqkb6mi3bz2u', 1],
+                        [u'fu7pazf6ogavkqj6z4q5qqex3u', 4],
+                        [u'fu7pazf6ogavkqj6z4q5qqex3u', 1],
+                        [u'hbyjtqvpcimwxiyqbcbbdn2i4a', 4],
+                        [u'hbyjtqvpcimwxiyqbcbbdn2i4a', 1],
+                        [u'pmcjbdkbjdl26k3e6yja77femq', 4],
+                        [u'pmcjbdkbjdl26k3e6yja77femq', 1],
+                        [u'r6swof4v2uttbiiqwj5pi32cm4', 4],
+                        [u'r6swof4v2uttbiiqwj5pi32cm4', 1],
+                        [u't45v5akoktf53evc2fi6gwnv6y', 4],
+                        [u't45v5akoktf53evc2fi6gwnv6y', 1],
+                        [u'y6zb4faar3rdvn3e6pfg4wlotm', 4],
+                        [u'y6zb4faar3rdvn3e6pfg4wlotm', 1],
+                        [u'z3yghutvqoqbchjao4lndnrh3a', 4],
+                        [u'z3yghutvqoqbchjao4lndnrh3a', 1],
+                    ],
+                    u'lease-age-histogram': {
+                        "1641600,1728000": 78,
+                        "12441600,12528000": 78,
+                        "8640000,8726400": 32,
+                        "1814400,1900800": 1860,
+                        "2764800,2851200": 76,
+                        "11491200,11577600": 20,
+                        "10713600,10800000": 183,
+                        "47865600,47952000": 7,
+                        "3110400,3196800": 328,
+                        "10627200,10713600": 43,
+                        "45619200,45705600": 4,
+                        "12873600,12960000": 5,
+                        "7430400,7516800": 7228,
+                        "1555200,1641600": 492,
+                        "38880000,38966400": 3,
+                        "12528000,12614400": 193,
+                        "7344000,7430400": 12689,
+                        "2678400,2764800": 278,
+                        "2332800,2419200": 12,
+                        "9244800,9331200": 73,
+                        "12787200,12873600": 218,
+                        "49075200,49161600": 19,
+                        "10368000,10454400": 117,
+                        "4665600,4752000": 256,
+                        "7516800,7603200": 993,
+                        "42336000,42422400": 33,
+                        "10972800,11059200": 122,
+                        "39052800,39139200": 51,
+                        "12614400,12700800": 210,
+                        "7603200,7689600": 2004,
+                        "10540800,10627200": 16,
+                        "950400,1036800": 4435,
+                        "42076800,42163200": 4,
+                        "8812800,8899200": 57,
+                        "5788800,5875200": 954,
+                        "36374400,36460800": 3,
+                        "9331200,9417600": 12,
+                        "30499200,30585600": 5,
+                        "12700800,12787200": 25,
+                        "2073600,2160000": 388,
+                        "12960000,13046400": 8,
+                        "11923200,12009600": 89,
+                        "3369600,3456000": 79,
+                        "3196800,3283200": 628,
+                        "37497600,37584000": 11,
+                        "33436800,33523200": 7,
+                        "44928000,45014400": 2,
+                        "37929600,38016000": 3,
+                        "38966400,39052800": 61,
+                        "3283200,3369600": 86,
+                        "11750400,11836800": 7,
+                        "3801600,3888000": 32,
+                        "46310400,46396800": 1,
+                        "4838400,4924800": 386,
+                        "8208000,8294400": 38,
+                        "37411200,37497600": 4,
+                        "12009600,12096000": 329,
+                        "10454400,10540800": 1239,
+                        "40176000,40262400": 1,
+                        "3715200,3801600": 104,
+                        "44409600,44496000": 13,
+                        "38361600,38448000": 5,
+                        "12268800,12355200": 2,
+                        "28771200,28857600": 6,
+                        "41990400,42076800": 10,
+                        "2592000,2678400": 40,
+                    },
+                },
+                'current-cycle': None,
+                'last-complete-bucket': None,
+            }
+        )