Changeset c9047b1 in trunk

Timestamp:

2016-04-28T00:40:04Z (9 years ago)

Author:

Brian Warner <warner@…>

Branches:

master

Children:

bde22ad1

Parents:

e187fba (diff), ea473cd (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch PR#242: switch stats-gatherer to JSON

Files:

• docs/stats.rst (modified) (2 diffs)
• misc/operations_helpers/munin/tahoe-stats.plugin-conf (modified) (1 diff)
• misc/operations_helpers/munin/tahoe_stats (modified) (2 diffs)
• src/allmydata/stats.py (modified) (5 diffs)
• topfiles/PR242.docs (added)

docs/stats.rst ¶

@@ -310 +310 @@
 as described in :doc:`configuration`.
 
-Once running, the stats gatherer will create a standard python "pickle" file
-in $BASEDIR/stats.pickle . Once a minute, the gatherer will pull stats
-information from every connected node and write them into the pickle. The
-pickle will contain a dictionary, in which node identifiers (known as "tubid"
+Once running, the stats gatherer will create a standard JSON file in
+``$BASEDIR/stats.json``. Once a minute, the gatherer will pull stats
+information from every connected node and write them into the file. The file
+will contain a dictionary, in which node identifiers (known as "tubid"
 strings) are the keys, and the values are a dict with 'timestamp',
 'nickname', and 'stats' keys. d[tubid][stats] will contain the stats
 dictionary as made available at http://localhost:3456/statistics?t=json . The
-pickle file will only contain the most recent update from each node.
+file will only contain the most recent update from each node.
 
 Other tools can be built to examine these stats and render them into
@@ -336 +336 @@
 Most of the plugins are designed to pull stats from a single Tahoe node, and
 are configured with the e.g. http://localhost:3456/statistics?t=json URL. The
-"tahoe_stats" plugin is designed to read from the pickle file created by the
+"tahoe_stats" plugin is designed to read from the JSON file created by the
 stats-gatherer. Some plugins are to be used with the disk watcher, and a few
 (like tahoe_nodememory) are designed to watch the node processes directly

misc/operations_helpers/munin/tahoe-stats.plugin-conf ¶

@@ -1 +1 @@
 [tahoe_storage_allocated]
-env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.pickle
+env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.json
 [tahoe_storage_consumed]
-env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.pickle
+env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.json
 [tahoe_runtime_load_avg]
-env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.pickle
+env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.json
 [tahoe_runtime_load_peak]
-env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.pickle
+env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.json
 [tahoe_storage_bytes_added]
-env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.pickle
+env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.json
 [tahoe_storage_bytes_freed]
-env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.pickle
+env.statsfile /home/robk/trees/tahoe/stats_gatherer/stats.json

misc/operations_helpers/munin/tahoe_stats ¶

@@ -2 +2 @@
 
 import os
-import pickle
+import json
 import re
 import sys
@@ -429 +429 @@
 def open_stats(fname):
     f = open(fname, 'rb')
-    stats = pickle.load(f)
+    stats = json.load(f)
     f.close()
     return stats

src/allmydata/stats.py ¶

@@ -1 +1 @@
 
+import json
 import os
-import pickle
 import pprint
 import time
@@ -241 +241 @@
         pprint.pprint(stats)
 
-class PickleStatsGatherer(StdOutStatsGatherer):
+class JSONStatsGatherer(StdOutStatsGatherer):
     # inherit from StdOutStatsGatherer for connect/disconnect notifications
 
@@ -247 +247 @@
         self.verbose = verbose
         StatsGatherer.__init__(self, basedir)
-        self.picklefile = os.path.join(basedir, "stats.pickle")
-
-        if os.path.exists(self.picklefile):
-            f = open(self.picklefile, 'rb')
+        self.jsonfile = os.path.join(basedir, "stats.json")
+
+        if os.path.exists(self.jsonfile):
+            f = open(self.jsonfile, 'rb')
             try:
-                self.gathered_stats = pickle.load(f)
+                self.gathered_stats = json.load(f)
             except Exception:
-                print ("Error while attempting to load pickle file %s.\n"
-                       "You may need to restore this file from a backup, or delete it if no backup is available.\n" %
-                       quote_local_unicode_path(self.picklefile))
+                print ("Error while attempting to load stats file %s.\n"
+                       "You may need to restore this file from a backup,"
+                       " or delete it if no backup is available.\n" %
+                       quote_local_unicode_path(self.jsonfile))
                 raise
             f.close()
@@ -267 +268 @@
         s['nickname'] = nickname
         s['stats'] = stats
-        self.dump_pickle()
-
-    def dump_pickle(self):
-        tmp = "%s.tmp" % (self.picklefile,)
+        self.dump_json()
+
+    def dump_json(self):
+        tmp = "%s.tmp" % (self.jsonfile,)
         f = open(tmp, 'wb')
-        pickle.dump(self.gathered_stats, f)
+        json.dump(self.gathered_stats, f)
         f.close()
-        if os.path.exists(self.picklefile):
-            os.unlink(self.picklefile)
-        os.rename(tmp, self.picklefile)
+        if os.path.exists(self.jsonfile):
+            os.unlink(self.jsonfile)
+        os.rename(tmp, self.jsonfile)
 
 class StatsGathererService(service.MultiService):
@@ -291 +292 @@
         self.tub.setOption("expose-remote-exception-types", False)
 
-        self.stats_gatherer = PickleStatsGatherer(self.basedir, verbose)
+        self.stats_gatherer = JSONStatsGatherer(self.basedir, verbose)
         self.stats_gatherer.setServiceParent(self)