| 1 | import sys |
|---|
| 2 | import os.path, time |
|---|
| 3 | from six.moves import cStringIO as StringIO |
|---|
| 4 | from twisted.trial import unittest |
|---|
| 5 | |
|---|
| 6 | from allmydata.util import fileutil |
|---|
| 7 | from allmydata.util.encodingutil import listdir_unicode |
|---|
| 8 | from allmydata.scripts import backupdb |
|---|
| 9 | from .common_util import skip_if_cannot_represent_filename |
|---|
| 10 | |
|---|
| 11 | class BackupDB(unittest.TestCase): |
|---|
| 12 | def create(self, dbfile): |
|---|
| 13 | stderr = StringIO() |
|---|
| 14 | bdb = backupdb.get_backupdb(dbfile, stderr=stderr) |
|---|
| 15 | self.failUnless(bdb, "unable to create backupdb from %r" % (dbfile,)) |
|---|
| 16 | return bdb |
|---|
| 17 | |
|---|
| 18 | def test_basic(self): |
|---|
| 19 | self.basedir = basedir = os.path.join("backupdb", "create") |
|---|
| 20 | fileutil.make_dirs(basedir) |
|---|
| 21 | dbfile = os.path.join(basedir, "dbfile") |
|---|
| 22 | bdb = self.create(dbfile) |
|---|
| 23 | self.failUnlessEqual(bdb.VERSION, 2) |
|---|
| 24 | |
|---|
| 25 | def test_upgrade_v1_v2(self): |
|---|
| 26 | self.basedir = basedir = os.path.join("backupdb", "upgrade_v1_v2") |
|---|
| 27 | fileutil.make_dirs(basedir) |
|---|
| 28 | dbfile = os.path.join(basedir, "dbfile") |
|---|
| 29 | stderr = StringIO() |
|---|
| 30 | created = backupdb.get_backupdb(dbfile, stderr=stderr, |
|---|
| 31 | create_version=(backupdb.SCHEMA_v1, 1), |
|---|
| 32 | just_create=True) |
|---|
| 33 | self.failUnless(created, "unable to create v1 backupdb") |
|---|
| 34 | # now we should have a v1 database on disk |
|---|
| 35 | bdb = self.create(dbfile) |
|---|
| 36 | self.failUnlessEqual(bdb.VERSION, 2) |
|---|
| 37 | |
|---|
| 38 | def test_fail(self): |
|---|
| 39 | self.basedir = basedir = os.path.join("backupdb", "fail") |
|---|
| 40 | fileutil.make_dirs(basedir) |
|---|
| 41 | |
|---|
| 42 | # put a non-DB file in the way |
|---|
| 43 | not_a_db = ("I do not look like a sqlite database\n" + |
|---|
| 44 | "I'M NOT" * 1000) # OS-X sqlite-2.3.2 takes some convincing |
|---|
| 45 | self.writeto("not-a-database", not_a_db) |
|---|
| 46 | stderr_f = StringIO() |
|---|
| 47 | bdb = backupdb.get_backupdb(os.path.join(basedir, "not-a-database"), |
|---|
| 48 | stderr_f) |
|---|
| 49 | self.failUnlessEqual(bdb, None) |
|---|
| 50 | stderr = stderr_f.getvalue() |
|---|
| 51 | self.failUnlessIn("backupdb file is unusable", stderr) |
|---|
| 52 | # sqlite-3.19.3 says "file is encrypted or is not a database" |
|---|
| 53 | # sqlite-3.20.0 says "file is not a database" |
|---|
| 54 | self.failUnlessIn("is not a database", stderr) |
|---|
| 55 | |
|---|
| 56 | # put a directory in the way, to exercise a different error path |
|---|
| 57 | where = os.path.join(basedir, "roadblock-dir") |
|---|
| 58 | fileutil.make_dirs(where) |
|---|
| 59 | stderr_f = StringIO() |
|---|
| 60 | bdb = backupdb.get_backupdb(where, stderr_f) |
|---|
| 61 | self.failUnlessEqual(bdb, None) |
|---|
| 62 | stderr = stderr_f.getvalue() |
|---|
| 63 | # the error-message is different under PyPy ... not sure why? |
|---|
| 64 | if 'pypy' in sys.version.lower(): |
|---|
| 65 | self.failUnlessIn("Could not open database", stderr) |
|---|
| 66 | else: |
|---|
| 67 | self.failUnlessIn("unable to open database file", stderr) |
|---|
| 68 | |
|---|
| 69 | |
|---|
| 70 | def writeto(self, filename, data): |
|---|
| 71 | fn = os.path.join(self.basedir, unicode(filename)) |
|---|
| 72 | parentdir = os.path.dirname(fn) |
|---|
| 73 | fileutil.make_dirs(parentdir) |
|---|
| 74 | fileutil.write(fn, data) |
|---|
| 75 | return fn |
|---|
| 76 | |
|---|
| 77 | def test_check(self): |
|---|
| 78 | self.basedir = basedir = os.path.join("backupdb", "check") |
|---|
| 79 | fileutil.make_dirs(basedir) |
|---|
| 80 | dbfile = os.path.join(basedir, "dbfile") |
|---|
| 81 | bdb = self.create(dbfile) |
|---|
| 82 | |
|---|
| 83 | foo_fn = self.writeto("foo.txt", "foo.txt") |
|---|
| 84 | blah_fn = self.writeto("bar/blah.txt", "blah.txt") |
|---|
| 85 | |
|---|
| 86 | r = bdb.check_file(foo_fn) |
|---|
| 87 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 88 | r.did_upload("foo-cap") |
|---|
| 89 | |
|---|
| 90 | r = bdb.check_file(blah_fn) |
|---|
| 91 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 92 | r.did_upload("blah-cap") |
|---|
| 93 | |
|---|
| 94 | r = bdb.check_file(foo_fn) |
|---|
| 95 | self.failUnlessEqual(r.was_uploaded(), "foo-cap") |
|---|
| 96 | self.failUnlessEqual(type(r.was_uploaded()), str) |
|---|
| 97 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 98 | |
|---|
| 99 | time.sleep(1.0) # make sure the timestamp changes |
|---|
| 100 | self.writeto("foo.txt", "NEW") |
|---|
| 101 | |
|---|
| 102 | r = bdb.check_file(foo_fn) |
|---|
| 103 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 104 | r.did_upload("new-cap") |
|---|
| 105 | |
|---|
| 106 | r = bdb.check_file(foo_fn) |
|---|
| 107 | self.failUnlessEqual(r.was_uploaded(), "new-cap") |
|---|
| 108 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 109 | # if we spontaneously decide to upload it anyways, nothing should |
|---|
| 110 | # break |
|---|
| 111 | r.did_upload("new-cap") |
|---|
| 112 | |
|---|
| 113 | r = bdb.check_file(foo_fn, use_timestamps=False) |
|---|
| 114 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 115 | r.did_upload("new-cap") |
|---|
| 116 | |
|---|
| 117 | r = bdb.check_file(foo_fn) |
|---|
| 118 | self.failUnlessEqual(r.was_uploaded(), "new-cap") |
|---|
| 119 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 120 | |
|---|
| 121 | bdb.NO_CHECK_BEFORE = 0 |
|---|
| 122 | bdb.ALWAYS_CHECK_AFTER = 0.1 |
|---|
| 123 | |
|---|
| 124 | r = bdb.check_file(blah_fn) |
|---|
| 125 | self.failUnlessEqual(r.was_uploaded(), "blah-cap") |
|---|
| 126 | self.failUnlessEqual(r.should_check(), True) |
|---|
| 127 | r.did_check_healthy("results") # we know they're ignored for now |
|---|
| 128 | |
|---|
| 129 | bdb.NO_CHECK_BEFORE = 200 |
|---|
| 130 | bdb.ALWAYS_CHECK_AFTER = 400 |
|---|
| 131 | |
|---|
| 132 | r = bdb.check_file(blah_fn) |
|---|
| 133 | self.failUnlessEqual(r.was_uploaded(), "blah-cap") |
|---|
| 134 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 135 | |
|---|
| 136 | os.unlink(os.path.join(basedir, "foo.txt")) |
|---|
| 137 | fileutil.make_dirs(os.path.join(basedir, "foo.txt")) # file becomes dir |
|---|
| 138 | r = bdb.check_file(foo_fn) |
|---|
| 139 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 140 | |
|---|
| 141 | def test_wrong_version(self): |
|---|
| 142 | self.basedir = basedir = os.path.join("backupdb", "wrong_version") |
|---|
| 143 | fileutil.make_dirs(basedir) |
|---|
| 144 | |
|---|
| 145 | where = os.path.join(basedir, "tooold.db") |
|---|
| 146 | bdb = self.create(where) |
|---|
| 147 | # reach into the DB and make it old |
|---|
| 148 | bdb.cursor.execute("UPDATE version SET version=0") |
|---|
| 149 | bdb.connection.commit() |
|---|
| 150 | |
|---|
| 151 | # now the next time we open the database, it should be an unusable |
|---|
| 152 | # version |
|---|
| 153 | stderr_f = StringIO() |
|---|
| 154 | bdb = backupdb.get_backupdb(where, stderr_f) |
|---|
| 155 | self.failUnlessEqual(bdb, None) |
|---|
| 156 | stderr = stderr_f.getvalue() |
|---|
| 157 | self.failUnlessEqual(stderr.strip(), |
|---|
| 158 | "Unable to handle backupdb version 0") |
|---|
| 159 | |
|---|
| 160 | def test_directory(self): |
|---|
| 161 | self.basedir = basedir = os.path.join("backupdb", "directory") |
|---|
| 162 | fileutil.make_dirs(basedir) |
|---|
| 163 | dbfile = os.path.join(basedir, "dbfile") |
|---|
| 164 | bdb = self.create(dbfile) |
|---|
| 165 | |
|---|
| 166 | contents = {u"file1": "URI:CHK:blah1", |
|---|
| 167 | u"file2": "URI:CHK:blah2", |
|---|
| 168 | u"dir1": "URI:DIR2-CHK:baz2"} |
|---|
| 169 | r = bdb.check_directory(contents) |
|---|
| 170 | self.failUnless(isinstance(r, backupdb.DirectoryResult)) |
|---|
| 171 | self.failIf(r.was_created()) |
|---|
| 172 | dircap = "URI:DIR2-CHK:foo1" |
|---|
| 173 | r.did_create(dircap) |
|---|
| 174 | |
|---|
| 175 | r = bdb.check_directory(contents) |
|---|
| 176 | self.failUnless(r.was_created()) |
|---|
| 177 | self.failUnlessEqual(r.was_created(), dircap) |
|---|
| 178 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 179 | |
|---|
| 180 | # if we spontaneously decide to upload it anyways, nothing should |
|---|
| 181 | # break |
|---|
| 182 | r.did_create(dircap) |
|---|
| 183 | r = bdb.check_directory(contents) |
|---|
| 184 | self.failUnless(r.was_created()) |
|---|
| 185 | self.failUnlessEqual(r.was_created(), dircap) |
|---|
| 186 | self.failUnlessEqual(type(r.was_created()), str) |
|---|
| 187 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 188 | |
|---|
| 189 | bdb.NO_CHECK_BEFORE = 0 |
|---|
| 190 | bdb.ALWAYS_CHECK_AFTER = 0.1 |
|---|
| 191 | time.sleep(1.0) |
|---|
| 192 | |
|---|
| 193 | r = bdb.check_directory(contents) |
|---|
| 194 | self.failUnless(r.was_created()) |
|---|
| 195 | self.failUnlessEqual(r.was_created(), dircap) |
|---|
| 196 | self.failUnlessEqual(r.should_check(), True) |
|---|
| 197 | r.did_check_healthy("results") |
|---|
| 198 | |
|---|
| 199 | bdb.NO_CHECK_BEFORE = 200 |
|---|
| 200 | bdb.ALWAYS_CHECK_AFTER = 400 |
|---|
| 201 | |
|---|
| 202 | r = bdb.check_directory(contents) |
|---|
| 203 | self.failUnless(r.was_created()) |
|---|
| 204 | self.failUnlessEqual(r.was_created(), dircap) |
|---|
| 205 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 206 | |
|---|
| 207 | |
|---|
| 208 | contents2 = {u"file1": "URI:CHK:blah1", |
|---|
| 209 | u"dir1": "URI:DIR2-CHK:baz2"} |
|---|
| 210 | r = bdb.check_directory(contents2) |
|---|
| 211 | self.failIf(r.was_created()) |
|---|
| 212 | |
|---|
| 213 | contents3 = {u"file1": "URI:CHK:blah1", |
|---|
| 214 | u"file2": "URI:CHK:blah3", |
|---|
| 215 | u"dir1": "URI:DIR2-CHK:baz2"} |
|---|
| 216 | r = bdb.check_directory(contents3) |
|---|
| 217 | self.failIf(r.was_created()) |
|---|
| 218 | |
|---|
| 219 | def test_unicode(self): |
|---|
| 220 | skip_if_cannot_represent_filename(u"f\u00f6\u00f6.txt") |
|---|
| 221 | skip_if_cannot_represent_filename(u"b\u00e5r.txt") |
|---|
| 222 | |
|---|
| 223 | self.basedir = basedir = os.path.join("backupdb", "unicode") |
|---|
| 224 | fileutil.make_dirs(basedir) |
|---|
| 225 | dbfile = os.path.join(basedir, "dbfile") |
|---|
| 226 | bdb = self.create(dbfile) |
|---|
| 227 | |
|---|
| 228 | self.writeto(u"f\u00f6\u00f6.txt", "foo.txt") |
|---|
| 229 | files = [fn for fn in listdir_unicode(unicode(basedir)) if fn.endswith(".txt")] |
|---|
| 230 | self.failUnlessEqual(len(files), 1) |
|---|
| 231 | foo_fn = os.path.join(basedir, files[0]) |
|---|
| 232 | #print foo_fn, type(foo_fn) |
|---|
| 233 | |
|---|
| 234 | r = bdb.check_file(foo_fn) |
|---|
| 235 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 236 | r.did_upload("foo-cap") |
|---|
| 237 | |
|---|
| 238 | r = bdb.check_file(foo_fn) |
|---|
| 239 | self.failUnlessEqual(r.was_uploaded(), "foo-cap") |
|---|
| 240 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 241 | |
|---|
| 242 | bar_fn = self.writeto(u"b\u00e5r.txt", "bar.txt") |
|---|
| 243 | #print bar_fn, type(bar_fn) |
|---|
| 244 | |
|---|
| 245 | r = bdb.check_file(bar_fn) |
|---|
| 246 | self.failUnlessEqual(r.was_uploaded(), False) |
|---|
| 247 | r.did_upload("bar-cap") |
|---|
| 248 | |
|---|
| 249 | r = bdb.check_file(bar_fn) |
|---|
| 250 | self.failUnlessEqual(r.was_uploaded(), "bar-cap") |
|---|
| 251 | self.failUnlessEqual(r.should_check(), False) |
|---|
| 252 | |
|---|
