Changeset 4f717ba in trunk

Timestamp:

2016-09-13T00:15:52Z (9 years ago)

Author:

David Stainton <dstainton415@…>

Branches:

master

Children:

4ae574c

Parents:

a8899c8

git-author:

David Stainton <dstainton415@…> (2016-09-12 23:01:23)

git-committer:

David Stainton <dstainton415@…> (2016-09-13 00:15:52)

Message:

Add a multi-introducer implementation

this is based the previous work of warner, killyourtv and leif
fixes ticket #68

Location:

src/allmydata

Files:

• client.py (modified) (5 diffs)
• introducer/client.py (modified) (4 diffs)
• test/test_helper.py (modified) (1 diff)
• test/test_introducer.py (modified) (4 diffs)
• test/test_multi_introducers.py (added)
• test/web/test_web.py (modified) (6 diffs)
• web/root.py (modified) (3 diffs)
• web/welcome.xhtml (modified) (2 diffs)

src/allmydata/client.py ¶

@@ -186 +186 @@
         self.logSource="Client"
         self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
-        self.init_introducer_client()
+        self.init_introducer_clients()
         self.init_stats_provider()
         self.init_secrets()
@@ -235 +235 @@
         return seqnum, nonce
 
-    def init_introducer_client(self):
-        self.introducer_furl = self.get_config("client", "introducer.furl")
-        introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_cache.yaml"))
-        ic = IntroducerClient(self.tub, self.introducer_furl,
-                              self.nickname,
-                              str(allmydata.__full_version__),
-                              str(self.OLDEST_SUPPORTED_VERSION),
-                              self.get_app_versions(),
-                              self._sequencer, introducer_cache_filepath)
-        self.introducer_client = ic
-        ic.setServiceParent(self)
+    def init_introducer_clients(self):
+        self.introducer_clients = []
+        self.introducer_furls = []
+
+        introducers_yaml_filename = os.path.join(self.basedir, "private", "introducers.yaml")
+        introducers_filepath = FilePath(introducers_yaml_filename)
+
+        try:
+            with introducers_filepath.open() as f:
+                introducers_yaml = yamlutil.safe_load(f)
+                introducers = introducers_yaml.get("introducers", {})
+                log.msg("found %d introducers in private/introducers.yaml" %
+                        len(introducers))
+        except EnvironmentError:
+            introducers = {}
+
+        if "default" in introducers.keys():
+            raise ValueError("'default' introducer furl cannot be specified in introducers.yaml; please fix impossible configuration.")
+
+        # read furl from tahoe.cfg
+        tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None)
+        if tahoe_cfg_introducer_furl:
+            introducers[u'default'] = {'furl':tahoe_cfg_introducer_furl}
+
+        for petname, introducer in introducers.items():
+            introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_{}_cache.yaml".format(petname)))
+            ic = IntroducerClient(self.tub, introducer['furl'],
+                                  self.nickname,
+                                  str(allmydata.__full_version__),
+                                  str(self.OLDEST_SUPPORTED_VERSION),
+                                  self.get_app_versions(), self._sequencer, introducer_cache_filepath)
+            self.introducer_clients.append(ic)
+            self.introducer_furls.append(introducer['furl'])
+            ic.setServiceParent(self)
 
     def init_stats_provider(self):
@@ -366 +389 @@
                "permutation-seed-base32": self._init_permutation_seed(ss),
                }
-        self.introducer_client.publish("storage", ann, self._node_key)
+        for ic in self.introducer_clients:
+            ic.publish("storage", ann, self._node_key)
 
     def init_client(self):
@@ -423 +447 @@
         self.storage_broker = sb
         sb.setServiceParent(self)
-
-        sb.use_introducer(self.introducer_client)
+        for ic in self.introducer_clients:
+            sb.use_introducer(ic)
 
     def get_storage_broker(self):
@@ -575 +599 @@
         return self.encoding_params
 
+    def introducer_connection_statuses(self):
+        return [ic.connected_to_introducer() for ic in self.introducer_clients]
+
     def connected_to_introducer(self):
-        if self.introducer_client:
-            return self.introducer_client.connected_to_introducer()
-        return False
+        return any([ic.connected_to_introducer() for ic in self.introducer_clients])
 
     def get_renewal_secret(self): # this will go away

src/allmydata/introducer/client.py ¶

@@ -48 +48 @@
 
         self._publisher = None
+        self._since = None
 
         self._local_subscribers = [] # (servicename,cb,args,kwargs) tuples
@@ -141 +142 @@
             raise InsufficientVersionError("V2", publisher.version)
         self._publisher = publisher
+        self._since = int(time.time())
         publisher.notifyOnDisconnect(self._disconnected)
         self._maybe_publish()
@@ -148 +150 @@
         self.log("bummer, we've lost our connection to the introducer")
         self._publisher = None
+        self._since = int(time.time())
         self._subscriptions.clear()
 
@@ -326 +329 @@
     def connected_to_introducer(self):
         return bool(self._publisher)
+
+    def get_since(self):
+        return self._since
+
+    def get_last_received_data_time(self):
+        if self._publisher is None:
+            return None
+        else:
+            return self._publisher.getDataLastReceivedAt()

src/allmydata/test/test_helper.py ¶

@@ -82 +82 @@
 
 class FakeClient(service.MultiService):
+    introducer_clients = []
     DEFAULT_ENCODING_PARAMETERS = {"k":25,
                                    "happy": 75,

src/allmydata/test/test_introducer.py ¶

@@ -728 +728 @@
         fileutil.make_dirs(basedir)
         cache_filepath = FilePath(os.path.join(basedir, "private",
-                                               "introducer_cache.yaml"))
+                                               "introducer_default_cache.yaml"))
 
         # if storage is enabled, the Client will publish its storage server
@@ -742 +742 @@
 
         c = TahoeClient(basedir)
-        ic = c.introducer_client
+        ic = c.introducer_clients[0]
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
@@ -810 +810 @@
 
         c2 = TahoeClient(basedir)
-        c2.introducer_client._load_announcements()
+        c2.introducer_clients[0]._load_announcements()
         yield flushEventualQueue()
         self.assertEqual(c2.storage_broker.get_all_serverids(),
@@ -831 +831 @@
 
         c = TahoeClient(basedir)
-        ic = c.introducer_client
+        ic = c.introducer_clients[0]
         outbound = ic._outbound_announcements
         published = ic._published_announcements

src/allmydata/test/web/test_web.py ¶

@@ -229 +229 @@
         self.nodeid = "fake_nodeid"
         self.nickname = u"fake_nickname \u263A"
-        self.introducer_furl = "None"
+        self.introducer_furls = []
+        self.introducer_clients = []
         self.stats_provider = FakeStatsProvider()
         self._secret_holder = SecretHolder("lease secret", "convergence secret")
@@ -658 +659 @@
             def connected_to_introducer(self):
                 return self.connected
+            def get_since(self):
+                return 0
+            def get_last_received_data_time(self):
+                return 0
 
         d = defer.succeed(None)
@@ -663 +668 @@
         # introducer not connected, unguessable furl
         def _set_introducer_not_connected_unguessable(ign):
-            self.s.introducer_furl = "pb://someIntroducer/secret"
-            self.s.introducer_client = MockIntroducerClient(False)
+            self.s.introducer_furls = [ "pb://someIntroducer/secret" ]
+            self.s.introducer_clients = [ MockIntroducerClient(False) ]
             return self.GET("/")
         d.addCallback(_set_introducer_not_connected_unguessable)
@@ -671 +676 @@
             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
             self.failIfIn('pb://someIntroducer/secret', html)
-            self.failUnless(re.search('<img (alt="Disconnected" |src="img/connected-no.png" ){2}/>', html), res)
+            self.failUnless(re.search('<img (alt="Disconnected" |src="img/connected-no.png" ){2}/></div>[ ]*<div>No introducers connected</div>', html), res)
+
         d.addCallback(_check_introducer_not_connected_unguessable)
 
         # introducer connected, unguessable furl
         def _set_introducer_connected_unguessable(ign):
-            self.s.introducer_furl = "pb://someIntroducer/secret"
-            self.s.introducer_client = MockIntroducerClient(True)
+            self.s.introducer_furls = [ "pb://someIntroducer/secret" ]
+            self.s.introducer_clients = [ MockIntroducerClient(True) ]
             return self.GET("/")
         d.addCallback(_set_introducer_connected_unguessable)
@@ -684 +690 @@
             self.failUnlessIn('<div class="furl">pb://someIntroducer/[censored]</div>', html)
             self.failIfIn('pb://someIntroducer/secret', html)
-            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
+            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/></div>[ ]*<div>1 introducer connected</div>', html), res)
         d.addCallback(_check_introducer_connected_unguessable)
 
         # introducer connected, guessable furl
         def _set_introducer_connected_guessable(ign):
-            self.s.introducer_furl = "pb://someIntroducer/introducer"
-            self.s.introducer_client = MockIntroducerClient(True)
+            self.s.introducer_furls = [ "pb://someIntroducer/introducer" ]
+            self.s.introducer_clients = [ MockIntroducerClient(True) ]
             return self.GET("/")
         d.addCallback(_set_introducer_connected_guessable)
@@ -696 +702 @@
             html = res.replace('\n', ' ')
             self.failUnlessIn('<div class="furl">pb://someIntroducer/introducer</div>', html)
-            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/>', html), res)
+            self.failUnless(re.search('<img (src="img/connected-yes.png" |alt="Connected" ){2}/></div>[ ]*<div>1 introducer connected</div>', html), res)
         d.addCallback(_check_introducer_connected_guessable)
         return d

src/allmydata/web/root.py ¶

@@ -231 +231 @@
         return ctx.tag[ul]
 
-    def data_introducer_furl_prefix(self, ctx, data):
-        ifurl = self.client.introducer_furl
-        # trim off the secret swissnum
-        (prefix, _, swissnum) = ifurl.rpartition("/")
-        if not ifurl:
-            return None
-        if swissnum == "introducer":
-            return ifurl
+    def data_introducer_description(self, ctx, data):
+        connected_count = self.data_connected_introducers( ctx, data )
+        if connected_count == 0:
+            return "No introducers connected"
+        elif connected_count == 1:
+            return "1 introducer connected"
         else:
-            return "%s/[censored]" % (prefix,)
-
-    def data_introducer_description(self, ctx, data):
-        if self.data_connected_to_introducer(ctx, data) == "no":
-            return "Introducer not connected"
-        return "Introducer"
+            return "%s introducers connected" % (connected_count,)
+
+    def data_total_introducers(self, ctx, data):
+        return len(self.client.introducer_furls)
+
+    def data_connected_introducers(self, ctx, data):
+        return self.client.introducer_connection_statuses().count(True)
 
     def data_connected_to_introducer(self, ctx, data):
@@ -252 +251 @@
         return "no"
 
-    def data_connected_to_introducer_alt(self, ctx, data):
-        return self._connectedalts[self.data_connected_to_introducer(ctx, data)]
+    def data_connected_to_at_least_one_introducer(self, ctx, data):
+        if True in self.client.introducer_connection_statuses():
+            return "yes"
+        return "no"
+
+    def data_connected_to_at_least_one_introducer_alt(self, ctx, data):
+        return self._connectedalts[self.data_connected_to_at_least_one_introducer(ctx, data)]
+
+    # In case we configure multiple introducers
+    def data_introducers(self, ctx, data):
+        connection_statuses = self.client.introducer_connection_statuses()
+        s = []
+        furls = self.client.introducer_furls
+        for furl in furls:
+            if connection_statuses:
+                display_furl = furl
+                # trim off the secret swissnum
+                (prefix, _, swissnum) = furl.rpartition("/")
+                if swissnum != "introducer":
+                    display_furl = "%s/[censored]" % (prefix,)
+                i = furls.index(furl)
+                ic = self.client.introducer_clients[i]
+                s.append((display_furl, bool(connection_statuses[i]), ic))
+        s.sort()
+        return s
+
+    def render_introducers_row(self, ctx, s):
+        (furl, connected, ic) = s
+        service_connection_status = "yes" if connected else "no"
+
+        since = ic.get_since()
+        service_connection_status_rel_time = render_time_delta(since, self.now_fn())
+        service_connection_status_abs_time = render_time_attr(since)
+
+        last_received_data_time = ic.get_last_received_data_time()
+        last_received_data_rel_time = render_time_delta(last_received_data_time, self.now_fn())
+        last_received_data_abs_time = render_time_attr(last_received_data_time)
+
+        ctx.fillSlots("introducer_furl", "%s" % (furl))
+        ctx.fillSlots("service_connection_status", "%s" % (service_connection_status,))
+        ctx.fillSlots("service_connection_status_alt",
+            self._connectedalts[service_connection_status])
+        ctx.fillSlots("service_connection_status_abs_time", service_connection_status_abs_time)
+        ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time)
+        ctx.fillSlots("last_received_data_abs_time", last_received_data_abs_time)
+        ctx.fillSlots("last_received_data_rel_time", last_received_data_rel_time)
+        return ctx.tag
 
     def data_helper_furl_prefix(self, ctx, data):
@@ -338 +382 @@
         ctx.fillSlots("address", addr)
         ctx.fillSlots("service_connection_status", service_connection_status)
-        ctx.fillSlots("service_connection_status_alt", self._connectedalts[service_connection_status])
-        ctx.fillSlots("connected-bool", bool(rhost))
+        ctx.fillSlots("service_connection_status_alt",
+                      self._connectedalts[service_connection_status])
         ctx.fillSlots("service_connection_status_abs_time", service_connection_status_abs_time)
         ctx.fillSlots("service_connection_status_rel_time", service_connection_status_rel_time)

src/allmydata/web/welcome.xhtml ¶

@@ -139 +139 @@
                 <div>
                   <h3>
-                    <div class="status-indicator"><img><n:attr name="src">img/connected-<n:invisible n:render="string" n:data="connected_to_introducer" />.png</n:attr><n:attr name="alt"><n:invisible n:render="string" n:data="connected_to_introducer_alt" /></n:attr></img></div>
+                    <div class="status-indicator"><img><n:attr name="src">img/connected-<n:invisible n:render="string" n:data="connected_to_at_least_one_introducer" />.png</n:attr><n:attr name="alt"><n:invisible n:render="string" n:data="connected_to_at_least_one_introducer_alt" /></n:attr></img></div>
                     <div n:render="string" n:data="introducer_description" />
                   </h3>
-                  <div class="furl" n:render="string" n:data="introducer_furl_prefix" />
                 </div>
                 <div>
@@ -198 +197 @@
             <tr n:pattern="empty"><td colspan="5">You are not presently connected to any peers</td></tr>
           </table>
+          <div class="row-fluid">
+            <h2>Connected to <span n:render="string" n:data="connected_introducers" /> of <span n:render="string" n:data="total_introducers" /> introducers</h2>
+          </div>
+          <table class="table table-striped table-bordered peer-status" n:render="sequence" n:data="introducers">
+            <thead>
+              <tr n:pattern="header">
+                <td><h3>Address</h3></td>
+                <td><h3>Last&nbsp;RX</h3></td>
+              </tr>
+            </thead>
+            <tr n:pattern="item" n:render="introducers_row">
+              <td class="nickname-and-peerid">
+                <div class="status-indicator"><img><n:attr name="src">img/connected-<n:slot name="service_connection_status" />.png</n:attr><n:attr name="alt"><n:slot name="service_connection_status_alt" /></n:attr></img></div>
+                <a class="timestamp"><n:attr name="title"><n:slot name="service_connection_status_abs_time"/></n:attr><n:slot name="service_connection_status_rel_time"/></a>
+                <div class="furl"><n:slot name="introducer_furl"/></div>
+              </td>
+              <td class="service-last-received-data"><a class="timestamp"><n:attr name="title"><n:slot name="last_received_data_abs_time"/></n:attr><n:slot name="last_received_data_rel_time"/></a></td>
+            </tr>
+            <tr n:pattern="empty"><td colspan="2">No introducers are configured.</td></tr>
+          </table>
         </div><!--/span-->
       </div><!--/row-->