diff -rN -u old-tahoe/contrib/fuse/impl_c/blackmatch.py new-tahoe/contrib/fuse/impl_c/blackmatch.py --- old-tahoe/contrib/fuse/impl_c/blackmatch.py 2010-01-24 05:51:58.956000000 +0000 +++ new-tahoe/contrib/fuse/impl_c/blackmatch.py 2010-01-24 05:52:03.173000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/env python #----------------------------------------------------------------------------------------------- -from allmydata.uri import CHKFileURI, DirectoryURI, LiteralFileURI +from allmydata.uri import CHKFileURI, DirectoryURI, LiteralFileURI, is_literal_file_uri from allmydata.scripts.common_http import do_http as do_http_req from allmydata.util.hashutil import tagged_hash from allmydata.util.assertutil import precondition @@ -335,7 +335,7 @@ self.fname = self.tfs.cache.tmp_file(os.urandom(20)) if self.fnode is None: log('TFF: [%s] open() for write: no file node, creating new File %s' % (self.name, self.fname, )) - self.fnode = File(0, 'URI:LIT:') + self.fnode = File(0, LiteralFileURI.BASE_STRING) self.fnode.tmp_fname = self.fname # XXX kill this self.parent.add_child(self.name, self.fnode, {}) elif hasattr(self.fnode, 'tmp_fname'): @@ -362,7 +362,7 @@ self.fname = self.fnode.tmp_fname log('TFF: reopening(%s) for reading' % self.fname) else: - if uri.startswith("URI:LIT") or not self.tfs.async: + if is_literal_file_uri(uri) or not self.tfs.async: log('TFF: synchronously fetching file from cache for reading') self.fname = self.tfs.cache.get_file(uri) else: @@ -906,7 +906,7 @@ class TStat(fuse.Stat): # in fuse 0.2, these are set by fuse.Stat.__init__ - # in fuse 0.2-pre3 (hardy) they are not. badness unsues if they're missing + # in fuse 0.2-pre3 (hardy) they are not. badness ensues if they're missing st_mode = None st_ino = 0 st_dev = 0 @@ -1237,7 +1237,7 @@ def get_file(self, uri): self.log('get_file(%s)' % (uri,)) - if uri.startswith("URI:LIT"): + if is_literal_file_uri(uri): return self.get_literal(uri) else: return self.get_chk(uri, async=False) diff -rN -u old-tahoe/docs/frontends/webapi.txt new-tahoe/docs/frontends/webapi.txt --- old-tahoe/docs/frontends/webapi.txt 2010-01-24 05:51:59.214000000 +0000 +++ new-tahoe/docs/frontends/webapi.txt 2010-01-24 05:52:03.375000000 +0000 @@ -70,20 +70,20 @@ these tasks. In general, everything that can be done with a PUT or DELETE can also be done with a POST. -Tahoe's web API is designed for two different consumers. The first is a -program that needs to manipulate the virtual file system. Such programs are +Tahoe's web API is designed for two different kinds of consumer. The first is +a program that needs to manipulate the virtual file system. Such programs are expected to use the RESTful interface described above. The second is a human using a standard web browser to work with the filesystem. This user is given a series of HTML pages with links to download files, and forms that use POST actions to upload, rename, and delete files. When an error occurs, the HTTP response code will be set to an appropriate -400-series code (like 404 for an unknown childname, or 400 Gone when a file -is unrecoverable due to insufficient shares), and the HTTP response body will -usually contain a few lines of explanation as to the cause of the error and -possible responses. Unusual exceptions may result in a 500 Internal Server -Error as a catch-all, with a default response body will contain a -Nevow-generated HTML-ized representation of the Python exception stack trace +400-series code (like 404 Not Found for an unknown childname, or 400 Bad Request +when the parameters to a webapi operation are invalid), and the HTTP response +body will usually contain a few lines of explanation as to the cause of the +error and possible responses. Unusual exceptions may result in a +500 Internal Server Error as a catch-all, with a default response body containing +a Nevow-generated HTML-ized representation of the Python exception stack trace that caused the problem. CLI programs which want to copy the response body to stderr should provide an "Accept: text/plain" header to their requests to get a plain text stack trace instead. If the Accept header contains */*, or @@ -108,9 +108,9 @@ read- and write- caps, which start with "URI:SSK", and give access to mutable files. -(later versions of Tahoe will make these strings shorter, and will remove the +(Later versions of Tahoe will make these strings shorter, and will remove the unfortunate colons, which must be escaped when these caps are embedded in -URLs). +URLs.) To refer to any Tahoe object through the web API, you simply need to combine a prefix (which indicates the HTTP server to use) with the cap (which @@ -150,8 +150,12 @@ === Child Lookup === -Tahoe directories contain named children, just like directories in a regular -local filesystem. These children can be either files or subdirectories. +Tahoe directories contain named child entries, just like directories in a regular +local filesystem. These child entries, called "dirnodes", consist of a name, +metadata, a write slot, and a read slot. The write and read slots normally contain +a writecap and readcap referring to the same object, which can be either a file +or a subdirectory. The write slot may be empty (actually, both may be empty, +but that is unusual). If you have a Tahoe URL that refers to a directory, and want to reference a named child inside it, just append the child name to the URL. For example, if @@ -195,9 +199,9 @@ representable as such. All Tahoe operations that refer to existing files or directories must include -a suitable read- or write- cap in the URL: the wapi server won't add one +a suitable read- or write- cap in the URL: the webapi server won't add one for you. If you don't know the cap, you can't access the file. This allows -the security properties of Tahoe caps to be extended across the wapi +the security properties of Tahoe caps to be extended across the webapi interface. == Slow Operations, Progress, and Cancelling == @@ -271,7 +275,7 @@ since the operation completed) will remain valid for ten minutes. Many "slow" operations can begin to use unacceptable amounts of memory when -operation on large directory structures. The memory usage increases when the +operating on large directory structures. The memory usage increases when the ophandle is polled, as the results must be copied into a JSON string, sent over the wire, then parsed by a client. So, as an alternative, many "slow" operations have streaming equivalents. These equivalents do not use operation @@ -314,7 +318,7 @@ To use the /uri/$FILECAP form, $FILECAP be a write-cap for a mutable file. In the /uri/$DIRCAP/[SUBDIRS../]FILENAME form, if the target file is a - writable mutable file, that files contents will be overwritten in-place. If + writable mutable file, that file's contents will be overwritten in-place. If it is a read-cap for a mutable file, an error will occur. If it is an immutable file, the old file will be discarded, and a new one will be put in its place. @@ -333,7 +337,7 @@ PUT /uri This uploads a file, and produces a file-cap for the contents, but does not - attach the file into the virtual drive. No directories will be modified by + attach the file into the filesystem. No directories will be modified by this operation. The file-cap is returned as the body of the HTTP response. If "mutable=true" is in the query arguments, the operation will create a @@ -347,7 +351,7 @@ Create a new empty directory and return its write-cap as the HTTP response body. This does not make the newly created directory visible from the - virtual drive. The "PUT" operation is provided for backwards compatibility: + filesystem. The "PUT" operation is provided for backwards compatibility: new code should use POST. POST /uri?t=mkdir-with-children @@ -388,8 +392,29 @@ "linkcrtime": 1202777696.7564139, "linkmotime": 1202777696.7564139, } } } ] - } + } + For forward-compatibility, a mutable directory can also contain caps in + a format that is unknown to the webapi server. When such caps are retrieved + from a mutable directory in a "ro_uri" field, they will be prefixed with + the string "ro.", indicating that they must not be decoded without + checking that they are read-only. The "ro." prefix must not be stripped + off without performing this check. (Future versions of the webapi server + will perform it where necessary.) + + If both the "rw_uri" and "ro_uri" fields are present in a given PROPDICT, + and the webapi server recognizes the rw_uri as a write cap, then it will + reset the ro_uri to the corresponding read cap and discard the original + contents of ro_uri (in order to ensure that the two caps correspond to the + same object and that the ro_uri is in fact read-only). However this may not + happen for caps in a format unknown to the webapi server. Therefore, when + writing a directory the webapi client should ensure that the contents + of "rw_uri" and "ro_uri" for a given PROPDICT are a consistent + (write cap, read cap) pair if possible. If the webapi client only has + one cap and does not know whether it is a write cap or read cap, then + it is acceptable to set "rw_uri" to that cap and omit "ro_uri". The + client must not put a write cap into a "ro_uri" field. + Note that the webapi-using client application must not provide the "Content-Type: multipart/form-data" header that usually accompanies HTML form submissions, since the body is not formatted this way. Doing so will @@ -404,59 +429,95 @@ Like t=mkdir-with-children above, but the new directory will be deep-immutable. This means that the directory itself is immutable, and that - it can only contain deep-immutable objects, like immutable files, literal - files, and deep-immutable directories. A non-empty request body is - mandatory, since after the directory is created, it will not be possible to - add more children to it. + it can only contain objects that are treated as being deep-immutable, like + immutable files, literal files, and deep-immutable directories. + + For forward-compatibility, a deep-immutable directory can also contain caps + in a format that is unknown to the webapi server. When such caps are retrieved + from a deep-immutable directory in a "ro_uri" field, they will be prefixed + with the string "imm.", indicating that they must not be decoded without + checking that they are immutable. The "imm." prefix must not be stripped + off without performing this check. (Future versions of the webapi server + will perform it where necessary.) + + The cap for each child may be given either in the "rw_uri" or "ro_uri" + field of the PROPDICT (not both). If a cap is given in the "rw_uri" field, + then the webapi server will check that it is an immutable readcap of a + *known* format, and give an error if it is not. If a cap is given in the + "ro_uri" field, then the webapi server will still check whether known + caps are immutable, but for unknown caps it will simply assume that the + cap can be stored, as described above. Note that an attacker would be + able to store any cap in an immutable directory, so this check when + creating the directory is only to help non-malicious clients to avoid + accidentally giving away more authority than intended. + + A non-empty request body is mandatory, since after the directory is created, + it will not be possible to add more children to it. POST /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir PUT /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir Create new directories as necessary to make sure that the named target ($DIRCAP/SUBDIRS../SUBDIR) is a directory. This will create additional - intermediate directories as necessary. If the named target directory already - exists, this will make no changes to it. + intermediate mutable directories as necessary. If the named target directory + already exists, this will make no changes to it. If the final directory is created, it will be empty. - This will return an error if a blocking file is present at any of the parent - names, preventing the server from creating the necessary parent directory. + This operation will return an error if a blocking file is present at any of + the parent names, preventing the server from creating the necessary parent + directory, or if it would require changing an immutable directory. The write-cap of the new directory will be returned as the HTTP response body. POST /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir-with-children - Like above, but if the final directory is created, it will be populated with - initial children from the POST request body, as described above in the - /uri?t=mkdir-with-children operation. + Like /uri?t=mkdir-with-children, but the final directory is created as a + child of an existing mutable directory. This will create additional + intermediate mutable directories as necessary. If the final directory is + created, it will be populated with initial children from the POST request + body, as described above. POST /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir-immutable - Like above, but the final directory will be deep-immutable, with the - children specified as a JSON dictionary in the POST request body. + Like /uri?t=mkdir-immutable, but the final directory is created as a child + of an existing mutable directory. The final directory will be deep-immutable, + and will be populated with the children specified as a JSON dictionary in + the POST request body. + + In Tahoe 1.6 this operation creates intermediate mutable directories if + necessary, but that behaviour should not be relied on; see ticket #920. POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir&name=NAME - Create a new empty directory and attach it to the given existing directory. - This will create additional intermediate directories as necessary. + Create a new empty mutable directory and attach it to the given existing + directory. This will create additional intermediate directories as necessary. - The URL of this form points to the parent of the bottom-most new directory, - whereas the previous form has a URL that points directly to the bottom-most - new directory. + This operation will return an error if a blocking file is present at any of + the parent names, preventing the server from creating the necessary parent + directory, or if it would require changing any immutable directory. + + The URL of this operation points to the parent of the bottommost new directory, + whereas the /uri/$DIRCAP/[SUBDIRS../]SUBDIR?t=mkdir operation above has a URL + that points directly to the bottommost new directory. POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir-with-children&name=NAME - As above, but the new directory will be populated with initial children via - the POST request body, as described in /uri?t=mkdir-with-children above. + Like /uri/$DIRCAP/[SUBDIRS../]?t=mkdir&name=NAME, but the new directory will + be populated with initial children via the POST request body. Note that the name= argument must be passed as a queryarg, because the POST request body is used for the initial children JSON. POST /uri/$DIRCAP/[SUBDIRS../]?t=mkdir-immutable&name=NAME - As above, but the new directory will be deep-immutable, with the children - specified as a JSON dictionary in the POST request body. Again, the name= - argument must be passed as a queryarg. + Like /uri/$DIRCAP/[SUBDIRS../]?t=mkdir-with-children&name=NAME, but the + final directory will be deep-immutable. The children are specified as a + JSON dictionary in the POST request body. Again, the name= argument must be + passed as a queryarg. + + In Tahoe 1.6 this operation creates intermediate mutable directories if + necessary, but that behaviour should not be relied on; see ticket #920. === Get Information About A File Or Directory (as JSON) === @@ -546,7 +607,7 @@ Then the rw_uri field will be present in the information about a directory if and only if you have read-write access to that directory. The verify_uri - field will be presend if and only if the object has a verify-cap + field will be present if and only if the object has a verify-cap (non-distributed LIT files do not have verify-caps). ==== About the metadata ==== @@ -622,11 +683,11 @@ link points. 4. Also, quite apart from Tahoe, you might be confused about the meaning of - the 'ctime' in unix local filesystems, which people sometimes think means - file creation time, but which actually means, in unix local filesystems, the + the 'ctime' in UNIX local filesystems, which people sometimes think means + file creation time, but which actually means, in UNIX local filesystems, the most recent time that the file contents or the file metadata (such as owner, permission bits, extended attributes, etc.) has changed. Note that although - 'ctime' does not mean file creation time in Unix, it does mean link creation + 'ctime' does not mean file creation time in UNIX, it does mean link creation time in Tahoe, unless the "tahoe backup" command has been used on that link, in which case it means something about the local filesystem file which corresponds to the Tahoe file which is pointed at by the link. It means @@ -634,7 +695,6 @@ Windows) or file-contents-or-metadata-update-time of the local file (if "tahoe backup" was run on a different operating system). - === Attaching an existing File or Directory by its read- or write- cap === PUT /uri/$DIRCAP/[SUBDIRS../]CHILDNAME?t=uri @@ -658,10 +718,15 @@ if there is already an object at the given location, rather than overwriting the existing object. To allow the operation to overwrite a file, but return an error when trying to overwrite a directory, use - "replace=only-files" (this behavior is closer to the traditional unix "mv" + "replace=only-files" (this behavior is closer to the traditional UNIX "mv" command). Note that "true", "t", and "1" are all synonyms for "True", and "false", "f", and "0" are synonyms for "False", and the parameter is case-insensitive. + + Note that this operation does not take its child cap in the form of + separate "rw_uri" and "ro_uri" fields. Therefore, it cannot accept a + child cap in a format unknown to the webapi server, because the server + is not able to attenuate an unknown write cap to a read cap. === Adding multiple files or directories to a parent directory at once === @@ -679,7 +744,9 @@ "childinfo" is a dictionary that contains "rw_uri", "ro_uri", and "metadata" keys. You can take the output of "GET /uri/$DIRCAP1?t=json" and use it as the input to "POST /uri/$DIRCAP2?t=set_children" to make DIR2 - look very much like DIR1. + look very much like DIR1 (except for any existing children of DIR2 that + were not overwritten, and any existing "tahoe" metadata keys as described + below). When the set_children request contains a child name that already exists in the target directory, this command defaults to overwriting that child with @@ -721,7 +788,7 @@ The object will only become completely unreachable once 1: there are no reachable directories that reference it, and 2: nobody is holding a read- or write- cap to the object. (This behavior is very similar to the way - hardlinks and anonymous files work in traditional unix filesystems). + hardlinks and anonymous files work in traditional UNIX filesystems). This operation will not modify more than a single directory. Intermediate directories which were implicitly created by PUT or POST methods will *not* @@ -850,7 +917,7 @@ POST /uri?t=upload This uploads a file, and produces a file-cap for the contents, but does not - attach the file into the virtual drive. No directories will be modified by + attach the file into the filesystem. No directories will be modified by this operation. The file must be provided as the "file" field of an HTML encoded form body, @@ -880,9 +947,9 @@ POST /uri/$DIRCAP/[SUBDIRS../]?t=upload - This uploads a file, and attaches it as a new child of the given directory. - The file must be provided as the "file" field of an HTML encoded form body, - produced in response to an HTML form like this: + This uploads a file, and attaches it as a new child of the given directory, + which must be mutable. The file must be provided as the "file" field of an + HTML-encoded form body, produced in response to an HTML form like this:
@@ -925,9 +992,10 @@ POST /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=upload This also uploads a file and attaches it as a new child of the given - directory. It is a slight variant of the previous operation, as the URL - refers to the target file rather than the parent directory. It is otherwise - identical: this accepts mutable= and when_done= arguments too. + directory, which must be mutable. It is a slight variant of the previous + operation, as the URL refers to the target file rather than the parent + directory. It is otherwise identical: this accepts mutable= and when_done= + arguments too. POST /uri/$FILECAP?t=upload @@ -955,20 +1023,21 @@ POST /uri/$DIRCAP/[SUBDIRS../]?t=delete&name=CHILDNAME - This instructs the node to delete a child object (file or subdirectory) from - the given directory. Note that the entire subtree is removed. This is - somewhat like "rm -rf" (from the point of view of the parent), but other - references into the subtree will see that the child subdirectories are not - modified by this operation. Only the link from the given directory to its - child is severed. + This instructs the node to remove a child object (file or subdirectory) from + the given directory, which must be mutable. Note that the entire subtree is + unlinked from the parent. Unlike deleting a subdirectory in a UNIX local + filesystem, the subtree need not be empty; if it isn't, then other references + into the subtree will see that the child subdirectories are not modified by + this operation. Only the link from the given directory to its child is severed. === Renaming A Child === POST /uri/$DIRCAP/[SUBDIRS../]?t=rename&from_name=OLD&to_name=NEW - This instructs the node to rename a child of the given directory. This is - exactly the same as removing the child, then adding the same child-cap under - the new name. This operation cannot move the child to a different directory. + This instructs the node to rename a child of the given directory, which must + be mutable. This has a similar effect to removing the child, then adding the + same child-cap under the new name, except that it preserves metadata. This + operation cannot move the child to a different directory. This operation will replace any existing child of the new name, making it behave like the UNIX "mv -f" command. @@ -1590,7 +1659,7 @@ == Static Files in /public_html == -The wapi server will take any request for a URL that starts with /static +The webapi server will take any request for a URL that starts with /static and serve it from a configurable directory which defaults to $BASEDIR/public_html . This is configured by setting the "[node]web.static" value in $BASEDIR/tahoe.cfg . If this is left at the default value of @@ -1598,10 +1667,10 @@ served with the contents of the file $BASEDIR/public_html/subdir/foo.html . This can be useful to serve a javascript application which provides a -prettier front-end to the rest of the Tahoe wapi. +prettier front-end to the rest of the Tahoe webapi. -== safety and security issues -- names vs. URIs == +== Safety and security issues -- names vs. URIs == Summary: use explicit file- and dir- caps whenever possible, to reduce the potential for surprises when the filesystem structure is changed. @@ -1641,6 +1710,17 @@ child's name and the child's URI are included in the results of listing the parent directory, so it isn't any harder to use the URI for this purpose. +The read and write caps in a given directory node are separate URIs, and +can't be assumed to point to the same object even if they were retrieved in +the same operation (although the webapi server attempts to ensure this +in most cases). If you need to rely on that property, you should explicitly +verify it. More generally, you should not make assumptions about the +internal consistency of the contents of mutable directories. As a result +of the signatures on mutable object versions, it is guaranteed that a given +version was written in a single update, but -- as in the case of a file -- +the contents may have been chosen by a malicious writer in a way that is +designed to confuse applications that rely on their consistency. + In general, use names if you want "whatever object (whether file or directory) is found by following this name (or sequence of names) when my request reaches the server". Use URIs if you want "this particular object". @@ -1676,7 +1756,7 @@ Tahoe nodes implement internal serialization to make sure that a single Tahoe node cannot conflict with itself. For example, it is safe to issue two -directory modification requests to a single tahoe node's wapi server at the +directory modification requests to a single tahoe node's webapi server at the same time, because the Tahoe node will internally delay one of them until after the other has finished being applied. (This feature was introduced in Tahoe-1.1; back with Tahoe-1.0 the web client was responsible for serializing diff -rN -u old-tahoe/docs/logging.txt new-tahoe/docs/logging.txt --- old-tahoe/docs/logging.txt 2010-01-24 05:51:59.330000000 +0000 +++ new-tahoe/docs/logging.txt 2010-01-24 05:52:03.452000000 +0000 @@ -224,6 +224,9 @@ == Log Messages During Unit Tests == +*** WARNING: setting the environment variables below may cause some tests to *** +*** fail spuriously. See ticket #923 for the status of a fix for this problem. *** + If a test is failing and you aren't sure why, start by enabling FLOGTOTWISTED=1 like this: diff -rN -u old-tahoe/relnotes.txt new-tahoe/relnotes.txt --- old-tahoe/relnotes.txt 2010-01-24 05:52:00.882000000 +0000 +++ new-tahoe/relnotes.txt 2010-01-24 05:52:05.267000000 +0000 @@ -1,7 +1,7 @@ -ANNOUNCING Tahoe, the Lofty-Atmospheric Filesystem, v1.5 +ANNOUNCING Tahoe, the Lofty-Atmospheric Filesystem, v1.6 The Tahoe-LAFS team is pleased to announce the immediate -availability of version 1.5 of Tahoe, the Lofty Atmospheric +availability of version 1.6 of Tahoe, the Lofty Atmospheric File System. Tahoe-LAFS is the first cloud storage technology which offers @@ -29,15 +29,20 @@ COMPATIBILITY -Version 1.5 is fully compatible with the version 1 series of -Tahoe-LAFS. Files written by v1.5 clients can be read by -clients of all versions back to v1.0. v1.5 clients can read -files produced by clients of all versions since v1.0. v1.5 -servers can serve clients of all versions back to v1.0 and v1.5 +Version 1.6 is fully compatible with the version 1 series of +Tahoe-LAFS. Files written by v1.6 clients can be read by +clients of all versions back to v1.0. v1.6 clients can read +files produced by clients of all versions since v1.0. v1.6 +servers can serve clients of all versions back to v1.0 and v1.6 clients can use servers of all versions back to v1.0. -This is the sixth release in the version 1 series. The version -1 series of Tahoe-LAFS will be actively supported and +In addition, version 1.6 improves forward-compatibility with +planned future cap formats, allowing updates to a directory +containing both current and future caps, without loss of +information. + +This is the seventh major release in the version 1 series. The +version 1 series of Tahoe-LAFS will be actively supported and maintained for the forseeable future, and future versions of Tahoe-LAFS will retain the ability to read and write files compatible with Tahoe-LAFS v1. diff -rN -u old-tahoe/src/allmydata/client.py new-tahoe/src/allmydata/client.py --- old-tahoe/src/allmydata/client.py 2010-01-24 05:52:00.924000000 +0000 +++ new-tahoe/src/allmydata/client.py 2010-01-24 05:52:05.334000000 +0000 @@ -471,13 +471,16 @@ # dirnodes. The first takes a URI and produces a filenode or (new-style) # dirnode. The other three create brand-new filenodes/dirnodes. - def create_node_from_uri(self, writecap, readcap=None): - # this returns synchronously. - return self.nodemaker.create_from_cap(writecap, readcap) + def create_node_from_uri(self, write_uri, read_uri=None, deep_immutable=False, name=""): + # This returns synchronously. + # Note that it does *not* validate the write_uri and read_uri; instead we + # may get an opaque node if there were any problems. + return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) def create_dirnode(self, initial_children={}): d = self.nodemaker.create_new_mutable_directory(initial_children) return d + def create_immutable_dirnode(self, children, convergence=None): return self.nodemaker.create_immutable_directory(children, convergence) diff -rN -u old-tahoe/src/allmydata/control.py new-tahoe/src/allmydata/control.py --- old-tahoe/src/allmydata/control.py 2010-01-24 05:52:00.936000000 +0000 +++ new-tahoe/src/allmydata/control.py 2010-01-24 05:52:05.351000000 +0000 @@ -5,7 +5,7 @@ from twisted.internet import defer from twisted.internet.interfaces import IConsumer from foolscap.api import Referenceable -from allmydata.interfaces import RIControlClient +from allmydata.interfaces import RIControlClient, IFileNode from allmydata.util import fileutil, mathutil from allmydata.immutable import upload from twisted.python import log @@ -67,7 +67,9 @@ return d def remote_download_from_uri_to_file(self, uri, filename): - filenode = self.parent.create_node_from_uri(uri) + filenode = self.parent.create_node_from_uri(uri, name=filename) + if not IFileNode.providedBy(filenode): + raise AssertionError("The URI does not reference a file.") c = FileWritingConsumer(filename) d = filenode.read(c) d.addCallback(lambda res: filename) @@ -199,6 +201,8 @@ if i >= self.count: return n = self.parent.create_node_from_uri(self.uris[i]) + if not IFileNode.providedBy(n): + raise AssertionError("The URI does not reference a file.") if n.is_mutable(): d1 = n.download_best_version() else: diff -rN -u old-tahoe/src/allmydata/dirnode.py new-tahoe/src/allmydata/dirnode.py --- old-tahoe/src/allmydata/dirnode.py 2010-01-24 05:52:00.946000000 +0000 +++ new-tahoe/src/allmydata/dirnode.py 2010-01-24 05:52:05.362000000 +0000 @@ -5,13 +5,13 @@ from twisted.internet import defer from foolscap.api import fireEventually import simplejson -from allmydata.mutable.common import NotMutableError +from allmydata.mutable.common import NotWriteableError from allmydata.mutable.filenode import MutableFileNode -from allmydata.unknown import UnknownNode +from allmydata.unknown import UnknownNode, strip_prefix_for_ro from allmydata.interfaces import IFilesystemNode, IDirectoryNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, \ ExistingChildError, NoSuchChildError, ICheckable, IDeepCheckable, \ - CannotPackUnknownNodeError + MustBeDeepImmutableError, CapConstraintError from allmydata.check_results import DeepCheckResults, \ DeepCheckAndRepairResults from allmydata.monitor import Monitor @@ -23,6 +23,11 @@ from pycryptopp.cipher.aes import AES from allmydata.util.dictutil import AuxValueDict + +# TODO: {Deleter,MetadataSetter,Adder}.modify all start by unpacking the +# contents and end by repacking them. It might be better to apply them to +# the unpacked contents. + class Deleter: def __init__(self, node, name, must_exist=True): self.node = node @@ -40,6 +45,7 @@ new_contents = self.node._pack_contents(children) return new_contents + class MetadataSetter: def __init__(self, node, name, metadata): self.node = node @@ -75,6 +81,11 @@ for (name, (child, new_metadata)) in self.entries.iteritems(): precondition(isinstance(name, unicode), name) precondition(IFilesystemNode.providedBy(child), child) + + # Strictly speaking this is redundant because we would raise the + # error again in pack_children. + child.raise_error() + if name in children: if not self.overwrite: raise ExistingChildError("child '%s' already exists" % name) @@ -123,25 +134,21 @@ new_contents = self.node._pack_contents(children) return new_contents -def _encrypt_rwcap(filenode, rwcap): - assert isinstance(rwcap, str) +def _encrypt_rw_uri(filenode, rw_uri): + assert isinstance(rw_uri, str) writekey = filenode.get_writekey() if not writekey: return "" - salt = hashutil.mutable_rwcap_salt_hash(rwcap) + salt = hashutil.mutable_rwcap_salt_hash(rw_uri) key = hashutil.mutable_rwcap_key_hash(salt, writekey) cryptor = AES(key) - crypttext = cryptor.process(rwcap) + crypttext = cryptor.process(rw_uri) mac = hashutil.hmac(key, salt + crypttext) assert len(mac) == 32 return salt + crypttext + mac # The MAC is not checked by readers in Tahoe >= 1.3.0, but we still # produce it for the sake of older readers. -class MustBeDeepImmutable(Exception): - """You tried to add a non-deep-immutable node to a deep-immutable - directory.""" - def pack_children(filenode, children, deep_immutable=False): """Take a dict that maps: children[unicode_name] = (IFileSystemNode, metadata_dict) @@ -152,7 +159,7 @@ time. If deep_immutable is True, I will require that all my children are deeply - immutable, and will raise a MustBeDeepImmutable exception if not. + immutable, and will raise a MustBeDeepImmutableError if not. """ has_aux = isinstance(children, AuxValueDict) @@ -161,25 +168,25 @@ assert isinstance(name, unicode) entry = None (child, metadata) = children[name] - if deep_immutable and child.is_mutable(): - # TODO: consider adding IFileSystemNode.is_deep_immutable() - raise MustBeDeepImmutable("child '%s' is mutable" % (name,)) + child.raise_error() + if deep_immutable and not child.is_allowed_in_immutable_directory(): + raise MustBeDeepImmutableError("child '%s' is not allowed in an immutable directory" % (name,), name) if has_aux: entry = children.get_aux(name) if not entry: assert IFilesystemNode.providedBy(child), (name,child) assert isinstance(metadata, dict) - rwcap = child.get_uri() # might be RO if the child is not writeable - if rwcap is None: - rwcap = "" - assert isinstance(rwcap, str), rwcap - rocap = child.get_readonly_uri() - if rocap is None: - rocap = "" - assert isinstance(rocap, str), rocap + rw_uri = child.get_write_uri() + if rw_uri is None: + rw_uri = "" + assert isinstance(rw_uri, str), rw_uri + ro_uri = child.get_readonly_uri() + if ro_uri is None: + ro_uri = "" + assert isinstance(ro_uri, str), ro_uri entry = "".join([netstring(name.encode("utf-8")), - netstring(rocap), - netstring(_encrypt_rwcap(filenode, rwcap)), + netstring(strip_prefix_for_ro(ro_uri, deep_immutable)), + netstring(_encrypt_rw_uri(filenode, rw_uri)), netstring(simplejson.dumps(metadata))]) entries.append(netstring(entry)) return "".join(entries) @@ -230,38 +237,66 @@ plaintext = cryptor.process(crypttext) return plaintext - def _create_node(self, rwcap, rocap): - return self._nodemaker.create_from_cap(rwcap, rocap) + def _create_and_validate_node(self, rw_uri, ro_uri, name): + #print "mutable? %r\n" % self.is_mutable() + #print "_create_and_validate_node(rw_uri=%r, ro_uri=%r, name=%r)\n" % (rw_uri, ro_uri, name) + node = self._nodemaker.create_from_cap(rw_uri, ro_uri, + deep_immutable=not self.is_mutable(), + name=name) + node.raise_error() + return node def _unpack_contents(self, data): # the directory is serialized as a list of netstrings, one per child. - # Each child is serialized as a list of four netstrings: (name, - # rocap, rwcap, metadata), in which the name,rocap,metadata are in - # cleartext. The 'name' is UTF-8 encoded. The rwcap is formatted as: - # pack("16ss32s", iv, AES(H(writekey+iv), plaintextrwcap), mac) + # Each child is serialized as a list of four netstrings: (name, ro_uri, + # rwcapdata, metadata), in which the name, ro_uri, metadata are in + # cleartext. The 'name' is UTF-8 encoded. The rwcapdata is formatted as: + # pack("16ss32s", iv, AES(H(writekey+iv), plaintext_rw_uri), mac) assert isinstance(data, str), (repr(data), type(data)) # an empty directory is serialized as an empty string if data == "": return AuxValueDict() writeable = not self.is_readonly() + mutable = self.is_mutable() children = AuxValueDict() position = 0 while position < len(data): entries, position = split_netstring(data, 1, position) entry = entries[0] - (name, rocap, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) + (name, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) name = name.decode("utf-8") - rwcap = None + rw_uri = "" if writeable: - rwcap = self._decrypt_rwcapdata(rwcapdata) - if not rwcap: - rwcap = None # rwcap is None or a non-empty string - if not rocap: - rocap = None # rocap is None or a non-empty string - child = self._create_node(rwcap, rocap) - metadata = simplejson.loads(metadata_s) - assert isinstance(metadata, dict) - children.set_with_aux(name, (child, metadata), auxilliary=entry) + rw_uri = self._decrypt_rwcapdata(rwcapdata) + #print "mutable=%r, writeable=%r, rw_uri=%r, ro_uri=%r, name=%r" % (mutable, writeable, rw_uri, ro_uri, name) + + # Since the encryption uses CTR mode, it currently leaks the length of the + # plaintext rw_uri -- and therefore whether it is present, i.e. whether the + # dirnode is writable (ticket #925). By stripping spaces in Tahoe >= 1.6.0, + # we may make it easier for future versions to plug this leak. + rw_uri = rw_uri.strip(' ') + if not rw_uri: + rw_uri = None # rw_uri is None or a non-empty string + + # Treat ro_uri in the same way for consistency. + ro_uri = ro_uri.strip(' ') + if not ro_uri: + ro_uri = None # ro_uri is None or a non-empty string + + try: + child = self._create_and_validate_node(rw_uri, ro_uri, name) + #print "%r.is_allowed_in_immutable_directory() = %r" % (child, child.is_allowed_in_immutable_directory()) + if mutable or child.is_allowed_in_immutable_directory(): + metadata = simplejson.loads(metadata_s) + assert isinstance(metadata, dict) + children[name] = (child, metadata) + children.set_with_aux(name, (child, metadata), auxilliary=entry) + except CapConstraintError, e: + #print "unmet constraint: (%s, %s)" % (e.args[0], e.args[1].encode("utf-8")) + log.msg(format="unmet constraint on cap for child '%(name)s' unpacked from a directory:\n" + "%(message)s", message=e.args[0], name=e.args[1].encode("utf-8"), + facility="tahoe.webish", level=log.UNUSUAL) + return children def _pack_contents(self, children): @@ -270,21 +305,39 @@ def is_readonly(self): return self._node.is_readonly() + def is_mutable(self): return self._node.is_mutable() + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return not self._node.is_mutable() + + def raise_error(self): + pass + def get_uri(self): return self._uri.to_string() + def get_write_uri(self): + if self.is_readonly(): + return None + return self._uri.to_string() + def get_readonly_uri(self): return self._uri.get_readonly().to_string() def get_cap(self): return self._uri + def get_readcap(self): return self._uri.get_readonly() + def get_verify_cap(self): return self._uri.get_verify_cap() + def get_repair_cap(self): if self._node.is_readonly(): return None # readonly (mutable) dirnodes are not yet repairable @@ -350,7 +403,7 @@ def set_metadata_for(self, name, metadata): assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) assert isinstance(metadata, dict) s = MetadataSetter(self, name, metadata) d = self._node.modify(s.modify) @@ -398,14 +451,10 @@ precondition(isinstance(name, unicode), name) precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) - child_node = self._create_node(writecap, readcap) - if isinstance(child_node, UnknownNode): - # don't be willing to pack unknown nodes: we might accidentally - # put some write-authority into the rocap slot because we don't - # know how to diminish the URI they gave us. We don't even know - # if they gave us a readcap or a writecap. - msg = "cannot pack unknown node as child %s" % str(name) - raise CannotPackUnknownNodeError(msg) + + # We now allow packing unknown nodes, provided they are valid + # for this type of directory. + child_node = self._create_and_validate_node(writecap, readcap, name) d = self.set_node(name, child_node, metadata, overwrite) d.addCallback(lambda res: child_node) return d @@ -423,10 +472,10 @@ writecap, readcap, metadata = e precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) - child_node = self._create_node(writecap, readcap) - if isinstance(child_node, UnknownNode): - msg = "cannot pack unknown node as child %s" % str(name) - raise CannotPackUnknownNodeError(msg) + + # We now allow packing unknown nodes, provided they are valid + # for this type of directory. + child_node = self._create_and_validate_node(writecap, readcap, name) a.set_node(name, child_node, metadata) d = self._node.modify(a.modify) d.addCallback(lambda ign: self) @@ -439,12 +488,12 @@ same name. If this directory node is read-only, the Deferred will errback with a - NotMutableError.""" + NotWriteableError.""" precondition(IFilesystemNode.providedBy(child), child) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) assert isinstance(name, unicode) assert IFilesystemNode.providedBy(child), child a = Adder(self, overwrite=overwrite) @@ -456,7 +505,7 @@ def set_nodes(self, entries, overwrite=True): precondition(isinstance(entries, dict), entries) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) a = Adder(self, entries, overwrite=overwrite) d = self._node.modify(a.modify) d.addCallback(lambda res: self) @@ -470,10 +519,10 @@ the operation completes.""" assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) d = self._uploader.upload(uploadable) - d.addCallback(lambda results: results.uri) - d.addCallback(self._nodemaker.create_from_cap) + d.addCallback(lambda results: + self._create_and_validate_node(results.uri, None, name)) d.addCallback(lambda node: self.set_node(name, node, metadata, overwrite)) return d @@ -483,7 +532,7 @@ fires (with the node just removed) when the operation finishes.""" assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) deleter = Deleter(self, name) d = self._node.modify(deleter.modify) d.addCallback(lambda res: deleter.old_child) @@ -493,7 +542,7 @@ mutable=True): assert isinstance(name, unicode) if self.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) if mutable: d = self._nodemaker.create_new_mutable_directory(initial_children) else: @@ -515,7 +564,7 @@ Deferred that fires when the operation finishes.""" assert isinstance(current_child_name, unicode) if self.is_readonly() or new_parent.is_readonly(): - return defer.fail(NotMutableError()) + return defer.fail(NotWriteableError()) if new_child_name is None: new_child_name = current_child_name assert isinstance(new_child_name, unicode) diff -rN -u old-tahoe/src/allmydata/immutable/filenode.py new-tahoe/src/allmydata/immutable/filenode.py --- old-tahoe/src/allmydata/immutable/filenode.py 2010-01-24 05:52:01.109000000 +0000 +++ new-tahoe/src/allmydata/immutable/filenode.py 2010-01-24 05:52:05.564000000 +0000 @@ -17,6 +17,9 @@ class _ImmutableFileNodeBase(object): implements(IImmutableFileNode, ICheckable) + def get_write_uri(self): + return None + def get_readonly_uri(self): return self.get_uri() @@ -26,6 +29,15 @@ def is_readonly(self): return True + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return True + + def raise_error(self): + pass + def __hash__(self): return self.u.__hash__() def __eq__(self, other): diff -rN -u old-tahoe/src/allmydata/interfaces.py new-tahoe/src/allmydata/interfaces.py --- old-tahoe/src/allmydata/interfaces.py 2010-01-24 05:52:01.138000000 +0000 +++ new-tahoe/src/allmydata/interfaces.py 2010-01-24 05:52:05.593000000 +0000 @@ -426,6 +426,7 @@ """Return True if the data can be modified by *somebody* (perhaps someone who has a more powerful URI than this one).""" + # TODO: rename to get_read_cap() def get_readonly(): """Return another IURI instance, which represents a read-only form of this one. If is_readonly() is True, this returns self.""" @@ -456,7 +457,6 @@ class IDirnodeURI(Interface): """I am a URI which represents a dirnode.""" - class IFileURI(Interface): """I am a URI which represents a filenode.""" def get_size(): @@ -467,21 +467,28 @@ class IMutableFileURI(Interface): """I am a URI which represents a mutable filenode.""" + class IDirectoryURI(Interface): pass + class IReadonlyDirectoryURI(Interface): pass -class CannotPackUnknownNodeError(Exception): - """UnknownNodes (using filecaps from the future that we don't understand) - cannot yet be copied safely, so I refuse to copy them.""" - -class UnhandledCapTypeError(Exception): - """I recognize the cap/URI, but I cannot create an IFilesystemNode for - it.""" +class CapConstraintError(Exception): + """A constraint on a cap was violated.""" -class NotDeepImmutableError(Exception): - """Deep-immutable directories can only contain deep-immutable children""" +class MustBeDeepImmutableError(CapConstraintError): + """Mutable children cannot be added to an immutable directory. + Also, caps obtained from an immutable directory can trigger this error + if they are later found to refer to a mutable object and then used.""" + +class MustBeReadonlyError(CapConstraintError): + """Known write caps cannot be specified in a ro_uri field. Also, + caps obtained from a ro_uri field can trigger this error if they + are later found to be write caps and then used.""" + +class MustNotBeUnknownRWError(CapConstraintError): + """Cannot add an unknown child cap specified in a rw_uri field.""" # The hierarchy looks like this: # IFilesystemNode @@ -518,9 +525,8 @@ """ def get_uri(): - """ - Return the URI string that can be used by others to get access to - this node. If this node is read-only, the URI will only offer + """Return the URI string corresponding to the strongest cap associated + with this node. If this node is read-only, the URI will only offer read-only access. If this node is read-write, the URI will offer read-write access. @@ -528,6 +534,11 @@ read-only access with others, use get_readonly_uri(). """ + def get_write_uri(n): + """Return the URI string that can be used by others to get write + access to this node, if it is writeable. If this is a read-only node, + return None.""" + def get_readonly_uri(): """Return the URI string that can be used by others to get read-only access to this node. The result is a read-only URI, regardless of @@ -557,6 +568,18 @@ file. """ + def is_unknown(): + """Return True if this is an unknown node.""" + + def is_allowed_in_immutable_directory(): + """Return True if this node is allowed as a child of a deep-immutable + directory. This is true if either the node is of a known-immutable type, + or it is unknown and read-only. + """ + + def raise_error(): + """Raise any error associated with this node.""" + def get_size(): """Return the length (in bytes) of the data this node represents. For directory nodes, I return the size of the backing store. I return @@ -902,7 +925,7 @@ ctime/mtime semantics of traditional filesystems. If this directory node is read-only, the Deferred will errback with a - NotMutableError.""" + NotWriteableError.""" def set_children(entries, overwrite=True): """Add multiple children (by writecap+readcap) to a directory node. @@ -928,7 +951,7 @@ ctime/mtime semantics of traditional filesystems. If this directory node is read-only, the Deferred will errback with a - NotMutableError.""" + NotWriteableError.""" def set_nodes(entries, overwrite=True): """Add multiple children to a directory node. Takes a dict mapping @@ -2074,7 +2097,7 @@ Tahoe process will typically have a single NodeMaker, but unit tests may create simplified/mocked forms for testing purposes. """ - def create_from_cap(writecap, readcap=None): + def create_from_cap(writecap, readcap=None, **kwargs): """I create an IFilesystemNode from the given writecap/readcap. I can only provide nodes for existing file/directory objects: use my other methods to create new objects. I return synchronously.""" diff -rN -u old-tahoe/src/allmydata/mutable/common.py new-tahoe/src/allmydata/mutable/common.py --- old-tahoe/src/allmydata/mutable/common.py 2010-01-24 05:52:01.186000000 +0000 +++ new-tahoe/src/allmydata/mutable/common.py 2010-01-24 05:52:05.658000000 +0000 @@ -8,7 +8,7 @@ # creation MODE_READ = "MODE_READ" -class NotMutableError(Exception): +class NotWriteableError(Exception): pass class NeedMoreDataError(Exception): diff -rN -u old-tahoe/src/allmydata/mutable/filenode.py new-tahoe/src/allmydata/mutable/filenode.py --- old-tahoe/src/allmydata/mutable/filenode.py 2010-01-24 05:52:01.191000000 +0000 +++ new-tahoe/src/allmydata/mutable/filenode.py 2010-01-24 05:52:05.665000000 +0000 @@ -214,6 +214,12 @@ def get_uri(self): return self._uri.to_string() + + def get_write_uri(self): + if self.is_readonly(): + return None + return self._uri.to_string() + def get_readonly_uri(self): return self._uri.get_readonly().to_string() @@ -227,9 +233,19 @@ def is_mutable(self): return self._uri.is_mutable() + def is_readonly(self): return self._uri.is_readonly() + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return not self._uri.is_mutable() + + def raise_error(self): + pass + def __hash__(self): return hash((self.__class__, self._uri)) def __cmp__(self, them): diff -rN -u old-tahoe/src/allmydata/nodemaker.py new-tahoe/src/allmydata/nodemaker.py --- old-tahoe/src/allmydata/nodemaker.py 2010-01-24 05:52:01.249000000 +0000 +++ new-tahoe/src/allmydata/nodemaker.py 2010-01-24 05:52:05.708000000 +0000 @@ -1,7 +1,7 @@ import weakref from zope.interface import implements from allmydata.util.assertutil import precondition -from allmydata.interfaces import INodeMaker, NotDeepImmutableError +from allmydata.interfaces import INodeMaker, MustBeDeepImmutableError from allmydata.immutable.filenode import ImmutableFileNode, LiteralFileNode from allmydata.immutable.upload import Data from allmydata.mutable.filenode import MutableFileNode @@ -44,28 +44,36 @@ def _create_dirnode(self, filenode): return DirectoryNode(filenode, self, self.uploader) - def create_from_cap(self, writecap, readcap=None): + def create_from_cap(self, writecap, readcap=None, deep_immutable=False, name=u""): # this returns synchronously. It starts with a "cap string". assert isinstance(writecap, (str, type(None))), type(writecap) assert isinstance(readcap, (str, type(None))), type(readcap) + #import traceback + #traceback.print_stack() + #print '%r.create_from_cap(%r, %r, %r)' % (self, writecap, readcap, kwargs) + bigcap = writecap or readcap if not bigcap: # maybe the writecap was hidden because we're in a readonly # directory, and the future cap format doesn't have a readcap, or # something. - return UnknownNode(writecap, readcap) - if bigcap in self._node_cache: - return self._node_cache[bigcap] - cap = uri.from_string(bigcap) - node = self._create_from_cap(cap) + return UnknownNode(None, None) # deep_immutable and name not needed + + # The name doesn't matter for caching since it's only used in the error + # attribute of an UnknownNode, and we don't cache those. + memokey = ("I" if deep_immutable else "M") + bigcap + if memokey in self._node_cache: + return self._node_cache[memokey] + cap = uri.from_string(bigcap, deep_immutable=deep_immutable, name=name) + node = self._create_from_single_cap(cap) if node: - self._node_cache[bigcap] = node # note: WeakValueDictionary + self._node_cache[memokey] = node # note: WeakValueDictionary else: - node = UnknownNode(writecap, readcap) # don't cache UnknownNode + # don't cache UnknownNode + node = UnknownNode(writecap, readcap, deep_immutable=deep_immutable, name=name) return node - def _create_from_cap(self, cap): - # This starts with a "cap instance" + def _create_from_single_cap(self, cap): if isinstance(cap, uri.LiteralFileURI): return self._create_lit(cap) if isinstance(cap, uri.CHKFileURI): @@ -76,7 +84,7 @@ uri.ReadonlyDirectoryURI, uri.ImmutableDirectoryURI, uri.LiteralDirectoryURI)): - filenode = self._create_from_cap(cap.get_filenode_cap()) + filenode = self._create_from_single_cap(cap.get_filenode_cap()) return self._create_dirnode(filenode) return None @@ -89,13 +97,11 @@ return d def create_new_mutable_directory(self, initial_children={}): - # initial_children must have metadata (i.e. {} instead of None), and - # should not contain UnknownNodes + # initial_children must have metadata (i.e. {} instead of None) for (name, (node, metadata)) in initial_children.iteritems(): - precondition(not isinstance(node, UnknownNode), - "create_new_mutable_directory does not accept UnknownNode", node) precondition(isinstance(metadata, dict), "create_new_mutable_directory requires metadata to be a dict, not None", metadata) + node.raise_error() d = self.create_mutable_file(lambda n: pack_children(n, initial_children)) d.addCallback(self._create_dirnode) @@ -105,19 +111,15 @@ if convergence is None: convergence = self.secret_holder.get_convergence_secret() for (name, (node, metadata)) in children.iteritems(): - precondition(not isinstance(node, UnknownNode), - "create_immutable_directory does not accept UnknownNode", node) precondition(isinstance(metadata, dict), "create_immutable_directory requires metadata to be a dict, not None", metadata) - if node.is_mutable(): - raise NotDeepImmutableError("%s is not immutable" % (node,)) + node.raise_error() + if not node.is_allowed_in_immutable_directory(): + raise MustBeDeepImmutableError("%s is not immutable" % (node,), name) n = DummyImmutableFileNode() # writekey=None packed = pack_children(n, children) uploadable = Data(packed, convergence) d = self.uploader.upload(uploadable, history=self.history) - def _uploaded(results): - filecap = self.create_from_cap(results.uri) - return filecap - d.addCallback(_uploaded) + d.addCallback(lambda results: self.create_from_cap(None, results.uri)) d.addCallback(self._create_dirnode) return d diff -rN -u old-tahoe/src/allmydata/scripts/common.py new-tahoe/src/allmydata/scripts/common.py --- old-tahoe/src/allmydata/scripts/common.py 2010-01-24 05:52:01.296000000 +0000 +++ new-tahoe/src/allmydata/scripts/common.py 2010-01-24 05:52:05.785000000 +0000 @@ -128,12 +128,14 @@ pass def get_alias(aliases, path, default): + from allmydata import uri # transform "work:path/filename" into (aliases["work"], "path/filename"). # If default=None, then an empty alias is indicated by returning - # DefaultAliasMarker. We special-case "URI:" to make it easy to access - # specific files/directories by their read-cap. + # DefaultAliasMarker. We special-case strings with a recognized cap URI + # prefix, to make it easy to access specific files/directories by their + # caps. path = path.strip() - if path.startswith("URI:"): + if uri.has_uri_prefix(path): # The only way to get a sub-path is to use URI:blah:./foo, and we # strip out the :./ sequence. sep = path.find(":./") diff -rN -u old-tahoe/src/allmydata/scripts/tahoe_cp.py new-tahoe/src/allmydata/scripts/tahoe_cp.py --- old-tahoe/src/allmydata/scripts/tahoe_cp.py 2010-01-24 05:52:01.358000000 +0000 +++ new-tahoe/src/allmydata/scripts/tahoe_cp.py 2010-01-24 05:52:05.845000000 +0000 @@ -258,8 +258,7 @@ readcap = ascii_or_none(data[1].get("ro_uri")) self.children[name] = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) - else: - assert data[0] == "dirnode" + elif data[0] == "dirnode": writecap = ascii_or_none(data[1].get("rw_uri")) readcap = ascii_or_none(data[1].get("ro_uri")) if writecap and writecap in self.cache: @@ -277,6 +276,11 @@ if recurse: child.populate(True) self.children[name] = child + else: + # TODO: there should be an option to skip unknown nodes. + raise TahoeError("Cannot copy unknown nodes (ticket #839). " + "You probably need to use a later version of " + "Tahoe-LAFS to copy this directory.") class TahoeMissingTarget: def __init__(self, url): @@ -353,8 +357,7 @@ urllib.quote(name.encode('utf-8'))]) self.children[name] = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) - else: - assert data[0] == "dirnode" + elif data[0] == "dirnode": writecap = ascii_or_none(data[1].get("rw_uri")) readcap = ascii_or_none(data[1].get("ro_uri")) if writecap and writecap in self.cache: @@ -372,6 +375,11 @@ if recurse: child.populate(True) self.children[name] = child + else: + # TODO: there should be an option to skip unknown nodes. + raise TahoeError("Cannot copy unknown nodes (ticket #839). " + "You probably need to use a later version of " + "Tahoe-LAFS to copy this directory.") def get_child_target(self, name): # return a new target for a named subdirectory of this dir @@ -407,9 +415,11 @@ set_data = {} for (name, filecap) in self.new_children.items(): # it just so happens that ?t=set_children will accept both file - # read-caps and write-caps as ['rw_uri'], and will handle eithe + # read-caps and write-caps as ['rw_uri'], and will handle either # correctly. So don't bother trying to figure out whether the one # we have is read-only or read-write. + # TODO: think about how this affects forward-compatibility for + # unknown caps set_data[name] = ["filenode", {"rw_uri": filecap}] body = simplejson.dumps(set_data) POST(url, body) @@ -770,6 +780,7 @@ # local-file-in-the-way # touch proposed # tahoe cp -r my:docs/proposed/denver.txt proposed/denver.txt +# handling of unknown nodes # things that maybe should be errors but aren't # local-dir-in-the-way diff -rN -u old-tahoe/src/allmydata/scripts/tahoe_put.py new-tahoe/src/allmydata/scripts/tahoe_put.py --- old-tahoe/src/allmydata/scripts/tahoe_put.py 2010-01-24 05:52:01.384000000 +0000 +++ new-tahoe/src/allmydata/scripts/tahoe_put.py 2010-01-24 05:52:05.867000000 +0000 @@ -40,6 +40,7 @@ # DIRCAP:./subdir/foo : DIRCAP/subdir/foo # MUTABLE-FILE-WRITECAP : filecap + # FIXME: this shouldn't rely on a particular prefix. if to_file.startswith("URI:SSK:"): url = nodeurl + "uri/%s" % urllib.quote(to_file) else: diff -rN -u old-tahoe/src/allmydata/test/common.py new-tahoe/src/allmydata/test/common.py --- old-tahoe/src/allmydata/test/common.py 2010-01-24 05:52:01.532000000 +0000 +++ new-tahoe/src/allmydata/test/common.py 2010-01-24 05:52:06.003000000 +0000 @@ -51,6 +51,8 @@ def get_uri(self): return self.my_uri.to_string() + def get_write_uri(self): + return None def get_readonly_uri(self): return self.my_uri.to_string() def get_cap(self): @@ -103,6 +105,12 @@ return False def is_readonly(self): return True + def is_unknown(self): + return False + def is_allowed_in_immutable_directory(self): + return True + def raise_error(self): + pass def get_size(self): try: @@ -190,6 +198,10 @@ return self.my_uri.get_readonly() def get_uri(self): return self.my_uri.to_string() + def get_write_uri(self): + if self.is_readonly(): + return None + return self.my_uri.to_string() def get_readonly(self): return self.my_uri.get_readonly() def get_readonly_uri(self): @@ -200,6 +212,12 @@ return self.my_uri.is_readonly() def is_mutable(self): return self.my_uri.is_mutable() + def is_unknown(self): + return False + def is_allowed_in_immutable_directory(self): + return not self.my_uri.is_mutable() + def raise_error(self): + pass def get_writekey(self): return "\x00"*16 def get_size(self): diff -rN -u old-tahoe/src/allmydata/test/test_client.py new-tahoe/src/allmydata/test/test_client.py --- old-tahoe/src/allmydata/test/test_client.py 2010-01-24 05:52:01.647000000 +0000 +++ new-tahoe/src/allmydata/test/test_client.py 2010-01-24 05:52:06.135000000 +0000 @@ -288,11 +288,14 @@ self.failUnless(n.is_readonly()) self.failUnless(n.is_mutable()) - future = "x-tahoe-crazy://future_cap_format." - n = c.create_node_from_uri(future) + unknown_rw = "lafs://from_the_future" + unknown_ro = "lafs://readonly_from_the_future" + n = c.create_node_from_uri(unknown_rw, unknown_ro) self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) self.failIf(IMutableFileNode.providedBy(n)) self.failIf(IDirectoryNode.providedBy(n)) - self.failUnlessEqual(n.get_uri(), future) + self.failUnless(n.is_unknown()) + self.failUnlessEqual(n.get_uri(), unknown_rw) + self.failUnlessEqual(n.get_readonly_uri(), "ro." + unknown_ro) diff -rN -u old-tahoe/src/allmydata/test/test_dirnode.py new-tahoe/src/allmydata/test/test_dirnode.py --- old-tahoe/src/allmydata/test/test_dirnode.py 2010-01-24 05:52:01.672000000 +0000 +++ new-tahoe/src/allmydata/test/test_dirnode.py 2010-01-24 05:52:06.160000000 +0000 @@ -7,8 +7,8 @@ from allmydata.client import Client from allmydata.immutable import upload from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ - ExistingChildError, NoSuchChildError, NotDeepImmutableError, \ - IDeepCheckResults, IDeepCheckAndRepairResults, CannotPackUnknownNodeError + ExistingChildError, NoSuchChildError, MustBeDeepImmutableError, \ + IDeepCheckResults, IDeepCheckAndRepairResults, MustNotBeUnknownRWError from allmydata.mutable.filenode import MutableFileNode from allmydata.mutable.common import UncoordinatedWriteError from allmydata.util import hashutil, base32 @@ -32,6 +32,11 @@ d = c.create_dirnode() def _done(res): self.failUnless(isinstance(res, dirnode.DirectoryNode)) + self.failUnless(res.is_mutable()) + self.failIf(res.is_readonly()) + self.failIf(res.is_unknown()) + self.failIf(res.is_allowed_in_immutable_directory()) + res.raise_error() rep = str(res) self.failUnless("RW-MUT" in rep) d.addCallback(_done) @@ -44,36 +49,74 @@ nm = c.nodemaker setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" one_uri = "URI:LIT:n5xgk" # LIT for "one" + mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), + u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}), + u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}), + u"fro": (nm.create_from_cap(None, future_read_uri), {}), } d = c.create_dirnode(kids) + def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failUnless(dn.is_mutable()) + self.failIf(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failIf(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RW-MUT" in rep) return dn.list() d.addCallback(_created) + def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) + self.failUnlessEqual(sorted(children.keys()), + [u"fro", u"fut", u"mut", u"one", u"two"]) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] + mut_node, mut_metadata = children[u"mut"] + fut_node, fut_metadata = children[u"fut"] + fro_node, fro_metadata = children[u"fro"] + self.failUnlessEqual(one_node.get_size(), 3) - self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(one_node.get_uri(), one_uri) + self.failUnlessEqual(one_node.get_readonly_uri(), one_uri) self.failUnless(isinstance(one_metadata, dict), one_metadata) + + self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(two_node.get_uri(), setup_py_uri) + self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri) self.failUnlessEqual(two_metadata["metakey"], "metavalue") + + self.failUnlessEqual(mut_node.get_uri(), mut_write_uri) + self.failUnlessEqual(mut_node.get_readonly_uri(), mut_read_uri) + self.failUnless(isinstance(mut_metadata, dict), mut_metadata) + + self.failUnless(fut_node.is_unknown()) + self.failUnlessEqual(fut_node.get_uri(), future_write_uri) + self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri) + self.failUnless(isinstance(fut_metadata, dict), fut_metadata) + + self.failUnless(fro_node.is_unknown()) + self.failUnlessEqual(fro_node.get_uri(), "ro." + future_read_uri) + self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri) + self.failUnless(isinstance(fro_metadata, dict), fro_metadata) d.addCallback(_check_kids) + d.addCallback(lambda ign: nm.create_new_mutable_directory(kids)) d.addCallback(lambda dn: dn.list()) d.addCallback(_check_kids) - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) - bad_kids1 = {u"one": (future_node, {})} + + bad_future_node = UnknownNode(future_write_uri, None) + bad_kids1 = {u"one": (bad_future_node, {})} d.addCallback(lambda ign: - self.shouldFail(AssertionError, "bad_kids1", - "does not accept UnknownNode", + self.shouldFail(MustNotBeUnknownRWError, "bad_kids1", + "cannot attach unknown", nm.create_new_mutable_directory, bad_kids1)) bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} @@ -91,17 +134,24 @@ nm = c.nodemaker setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" one_uri = "URI:LIT:n5xgk" # LIT for "one" - mut_readcap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" - mut_writecap = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), + u"fut": (nm.create_from_cap(None, future_read_uri), {}), } d = c.create_immutable_dirnode(kids) + def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failUnless(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() @@ -109,50 +159,73 @@ self.cap = cap return dn.list() d.addCallback(_created) + def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), [u"one", u"two"]) + self.failUnlessEqual(sorted(children.keys()), [u"fut", u"one", u"two"]) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] + fut_node, fut_metadata = children[u"fut"] + self.failUnlessEqual(one_node.get_size(), 3) - self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(one_node.get_uri(), one_uri) + self.failUnlessEqual(one_node.get_readonly_uri(), one_uri) self.failUnless(isinstance(one_metadata, dict), one_metadata) + + self.failUnlessEqual(two_node.get_size(), 14861) + self.failUnlessEqual(two_node.get_uri(), setup_py_uri) + self.failUnlessEqual(two_node.get_readonly_uri(), setup_py_uri) self.failUnlessEqual(two_metadata["metakey"], "metavalue") + + self.failUnless(fut_node.is_unknown()) + self.failUnlessEqual(fut_node.get_uri(), "imm." + future_read_uri) + self.failUnlessEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri) + self.failUnless(isinstance(fut_metadata, dict), fut_metadata) d.addCallback(_check_kids) + d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) d.addCallback(lambda dn: dn.list()) d.addCallback(_check_kids) - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) - bad_kids1 = {u"one": (future_node, {})} + + bad_future_node1 = UnknownNode(future_write_uri, None) + bad_kids1 = {u"one": (bad_future_node1, {})} d.addCallback(lambda ign: - self.shouldFail(AssertionError, "bad_kids1", - "does not accept UnknownNode", + self.shouldFail(MustNotBeUnknownRWError, "bad_kids1", + "cannot attach unknown", c.create_immutable_dirnode, bad_kids1)) - bad_kids2 = {u"one": (nm.create_from_cap(one_uri), None)} + bad_future_node2 = UnknownNode(future_write_uri, future_read_uri) + bad_kids2 = {u"one": (bad_future_node2, {})} d.addCallback(lambda ign: - self.shouldFail(AssertionError, "bad_kids2", - "requires metadata to be a dict", + self.shouldFail(MustBeDeepImmutableError, "bad_kids2", + "is not immutable", c.create_immutable_dirnode, bad_kids2)) - bad_kids3 = {u"one": (nm.create_from_cap(mut_writecap), {})} + bad_kids3 = {u"one": (nm.create_from_cap(one_uri), None)} d.addCallback(lambda ign: - self.shouldFail(NotDeepImmutableError, "bad_kids3", - "is not immutable", + self.shouldFail(AssertionError, "bad_kids3", + "requires metadata to be a dict", c.create_immutable_dirnode, bad_kids3)) - bad_kids4 = {u"one": (nm.create_from_cap(mut_readcap), {})} + bad_kids4 = {u"one": (nm.create_from_cap(mut_write_uri), {})} d.addCallback(lambda ign: - self.shouldFail(NotDeepImmutableError, "bad_kids4", + self.shouldFail(MustBeDeepImmutableError, "bad_kids4", "is not immutable", c.create_immutable_dirnode, bad_kids4)) + bad_kids5 = {u"one": (nm.create_from_cap(mut_read_uri), {})} + d.addCallback(lambda ign: + self.shouldFail(MustBeDeepImmutableError, "bad_kids5", + "is not immutable", + c.create_immutable_dirnode, + bad_kids5)) d.addCallback(lambda ign: c.create_immutable_dirnode({})) def _created_empty(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failUnless(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() @@ -168,6 +241,9 @@ self.failUnless(isinstance(dn, dirnode.DirectoryNode)) self.failIf(dn.is_mutable()) self.failUnless(dn.is_readonly()) + self.failIf(dn.is_unknown()) + self.failUnless(dn.is_allowed_in_immutable_directory()) + dn.raise_error() rep = str(dn) self.failUnless("RO-IMM" in rep) cap = dn.get_cap() @@ -193,9 +269,9 @@ d.addCallback(_check_kids) d.addCallback(lambda ign: n.get(u"subdir")) d.addCallback(lambda sd: self.failIf(sd.is_mutable())) - bad_kids = {u"one": (nm.create_from_cap(mut_writecap), {})} + bad_kids = {u"one": (nm.create_from_cap(mut_write_uri), {})} d.addCallback(lambda ign: - self.shouldFail(NotDeepImmutableError, "YZ", + self.shouldFail(MustBeDeepImmutableError, "YZ", "is not immutable", n.create_subdirectory, u"sub2", bad_kids, mutable=False)) @@ -203,7 +279,6 @@ d.addCallback(_made_parent) return d - def test_check(self): self.basedir = "dirnode/Dirnode/test_check" self.set_up_grid() @@ -337,24 +412,27 @@ ro_dn = c.create_node_from_uri(ro_uri) self.failUnless(ro_dn.is_readonly()) self.failUnless(ro_dn.is_mutable()) + self.failIf(ro_dn.is_unknown()) + self.failIf(ro_dn.is_allowed_in_immutable_directory()) + ro_dn.raise_error() - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.set_uri, u"newchild", filecap, filecap) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.set_node, u"newchild", filenode) - self.shouldFail(dirnode.NotMutableError, "set_nodes ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_nodes ro", None, ro_dn.set_nodes, { u"newchild": (filenode, None) }) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.add_file, u"newchild", uploadable) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.delete, u"child") - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.create_subdirectory, u"newchild") - self.shouldFail(dirnode.NotMutableError, "set_metadata_for ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_metadata_for ro", None, ro_dn.set_metadata_for, u"child", {}) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, ro_dn.move_child_to, u"child", rw_dn) - self.shouldFail(dirnode.NotMutableError, "set_uri ro", None, + self.shouldFail(dirnode.NotWriteableError, "set_uri ro", None, rw_dn.move_child_to, u"child", ro_dn) return ro_dn.list() d.addCallback(_ready) @@ -901,8 +979,8 @@ nodemaker = NodeMaker(None, None, None, None, None, None, {"k": 3, "n": 10}, None) - writecap = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" - filenode = nodemaker.create_from_cap(writecap) + write_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + filenode = nodemaker.create_from_cap(write_uri) node = dirnode.DirectoryNode(filenode, nodemaker, None) children = node._unpack_contents(known_tree) self._check_children(children) @@ -975,23 +1053,23 @@ self.failUnlessIn("lit", packed) kids = self._make_kids(nm, ["imm", "lit", "write"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) # read-only is not enough: all children must be immutable kids = self._make_kids(nm, ["imm", "lit", "read"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) kids = self._make_kids(nm, ["imm", "lit", "dirwrite"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) kids = self._make_kids(nm, ["imm", "lit", "dirread"]) - self.failUnlessRaises(dirnode.MustBeDeepImmutable, + self.failUnlessRaises(dirnode.MustBeDeepImmutableError, dirnode.pack_children, fn, kids, deep_immutable=True) @@ -1017,16 +1095,31 @@ def get_cap(self): return self.uri + def get_uri(self): return self.uri.to_string() + + def get_write_uri(self): + return self.uri.to_string() + def download_best_version(self): return defer.succeed(self.data) + def get_writekey(self): return "writekey" + def is_readonly(self): return False + def is_mutable(self): return True + + def is_unknown(self): + return False + + def is_allowed_in_immutable_directory(self): + return False + def modify(self, modifier): self.data = modifier(self.data, None, True) return defer.succeed(None) @@ -1050,47 +1143,59 @@ def test_from_future(self): # create a dirnode that contains unknown URI types, and make sure we - # tolerate them properly. Since dirnodes aren't allowed to add - # unknown node types, we have to be tricky. + # tolerate them properly. d = self.nodemaker.create_new_mutable_directory() - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_write_uri, future_read_uri) def _then(n): self._node = n return n.set_node(u"future", future_node) d.addCallback(_then) - # we should be prohibited from adding an unknown URI to a directory, - # since we don't know how to diminish the cap to a readcap (for the - # dirnode's rocap slot), and we don't want to accidentally grant - # write access to a holder of the dirnode's readcap. + # We should be prohibited from adding an unknown URI to a directory + # just in the rw_uri slot, since we don't know how to diminish the cap + # to a readcap (for the ro_uri slot). d.addCallback(lambda ign: - self.shouldFail(CannotPackUnknownNodeError, + self.shouldFail(MustNotBeUnknownRWError, "copy unknown", - "cannot pack unknown node as child add", + "cannot attach unknown rw cap as child", self._node.set_uri, u"add", - future_writecap, future_readcap)) + future_write_uri, None)) + + # However, we should be able to add both rw_uri and ro_uri as a pair of + # unknown URIs. + d.addCallback(lambda ign: self._node.set_uri(u"add-pair", + future_write_uri, future_read_uri)) + d.addCallback(lambda ign: self._node.list()) def _check(children): - self.failUnlessEqual(len(children), 1) + self.failUnlessEqual(len(children), 2) (fn, metadata) = children[u"future"] self.failUnless(isinstance(fn, UnknownNode), fn) - self.failUnlessEqual(fn.get_uri(), future_writecap) - self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) - # but we *should* be allowed to copy this node, because the + self.failUnlessEqual(fn.get_uri(), future_write_uri) + self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri) + + (fn2, metadata2) = children[u"add-pair"] + self.failUnless(isinstance(fn2, UnknownNode), fn2) + self.failUnlessEqual(fn2.get_uri(), future_write_uri) + self.failUnlessEqual(fn2.get_readonly_uri(), "ro." + future_read_uri) + + # we should also be allowed to copy this node, because the # UnknownNode contains all the information that was in the # original directory (readcap and writecap), so we're preserving # everything. return self._node.set_node(u"copy", fn) d.addCallback(_check) + d.addCallback(lambda ign: self._node.list()) def _check2(children): - self.failUnlessEqual(len(children), 2) + self.failUnlessEqual(len(children), 3) (fn, metadata) = children[u"copy"] self.failUnless(isinstance(fn, UnknownNode), fn) - self.failUnlessEqual(fn.get_uri(), future_writecap) - self.failUnlessEqual(fn.get_readonly_uri(), future_readcap) + self.failUnlessEqual(fn.get_uri(), future_write_uri) + self.failUnlessEqual(fn.get_readonly_uri(), "ro." + future_read_uri) + d.addCallback(_check2) return d class DeepStats(unittest.TestCase): diff -rN -u old-tahoe/src/allmydata/test/test_filenode.py new-tahoe/src/allmydata/test/test_filenode.py --- old-tahoe/src/allmydata/test/test_filenode.py 2010-01-24 05:52:01.687000000 +0000 +++ new-tahoe/src/allmydata/test/test_filenode.py 2010-01-24 05:52:06.173000000 +0000 @@ -41,14 +41,21 @@ self.failUnlessEqual(fn1.get_readcap(), u) self.failUnlessEqual(fn1.is_readonly(), True) self.failUnlessEqual(fn1.is_mutable(), False) + self.failUnlessEqual(fn1.is_unknown(), False) + self.failUnlessEqual(fn1.is_allowed_in_immutable_directory(), True) + self.failUnlessEqual(fn1.get_write_uri(), None) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_size(), 1000) self.failUnlessEqual(fn1.get_storage_index(), u.storage_index) + fn1.raise_error() + fn2.raise_error() d = {} d[fn1] = 1 # exercise __hash__ v = fn1.get_verify_cap() self.failUnless(isinstance(v, uri.CHKFileVerifierURI)) self.failUnlessEqual(fn1.get_repair_cap(), v) + self.failUnlessEqual(v.is_readonly(), True) + self.failUnlessEqual(v.is_mutable(), False) def test_literal_filenode(self): @@ -64,9 +71,14 @@ self.failUnlessEqual(fn1.get_readcap(), u) self.failUnlessEqual(fn1.is_readonly(), True) self.failUnlessEqual(fn1.is_mutable(), False) + self.failUnlessEqual(fn1.is_unknown(), False) + self.failUnlessEqual(fn1.is_allowed_in_immutable_directory(), True) + self.failUnlessEqual(fn1.get_write_uri(), None) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_size(), len(DATA)) self.failUnlessEqual(fn1.get_storage_index(), None) + fn1.raise_error() + fn2.raise_error() d = {} d[fn1] = 1 # exercise __hash__ @@ -99,24 +111,29 @@ self.failUnlessEqual(n.get_writekey(), wk) self.failUnlessEqual(n.get_readkey(), rk) self.failUnlessEqual(n.get_storage_index(), si) - # these itmes are populated on first read (or create), so until that + # these items are populated on first read (or create), so until that # happens they'll be None self.failUnlessEqual(n.get_privkey(), None) self.failUnlessEqual(n.get_encprivkey(), None) self.failUnlessEqual(n.get_pubkey(), None) self.failUnlessEqual(n.get_uri(), u.to_string()) + self.failUnlessEqual(n.get_write_uri(), u.to_string()) self.failUnlessEqual(n.get_readonly_uri(), u.get_readonly().to_string()) self.failUnlessEqual(n.get_cap(), u) self.failUnlessEqual(n.get_readcap(), u.get_readonly()) self.failUnlessEqual(n.is_mutable(), True) self.failUnlessEqual(n.is_readonly(), False) + self.failUnlessEqual(n.is_unknown(), False) + self.failUnlessEqual(n.is_allowed_in_immutable_directory(), False) + n.raise_error() n2 = MutableFileNode(None, None, client.get_encoding_parameters(), None).init_from_cap(u) self.failUnlessEqual(n, n2) self.failIfEqual(n, "not even the right type") self.failIfEqual(n, u) # not the right class + n.raise_error() d = {n: "can these be used as dictionary keys?"} d[n2] = "replace the old one" self.failUnlessEqual(len(d), 1) @@ -127,12 +144,16 @@ self.failUnlessEqual(nro.get_readonly(), nro) self.failUnlessEqual(nro.get_cap(), u.get_readonly()) self.failUnlessEqual(nro.get_readcap(), u.get_readonly()) + self.failUnlessEqual(nro.is_mutable(), True) + self.failUnlessEqual(nro.is_readonly(), True) + self.failUnlessEqual(nro.is_unknown(), False) + self.failUnlessEqual(nro.is_allowed_in_immutable_directory(), False) nro_u = nro.get_uri() self.failUnlessEqual(nro_u, nro.get_readonly_uri()) self.failUnlessEqual(nro_u, u.get_readonly().to_string()) - self.failUnlessEqual(nro.is_mutable(), True) - self.failUnlessEqual(nro.is_readonly(), True) + self.failUnlessEqual(nro.get_write_uri(), None) self.failUnlessEqual(nro.get_repair_cap(), None) # RSAmut needs writecap + nro.raise_error() v = n.get_verify_cap() self.failUnless(isinstance(v, uri.SSKVerifierURI)) diff -rN -u old-tahoe/src/allmydata/test/test_system.py new-tahoe/src/allmydata/test/test_system.py --- old-tahoe/src/allmydata/test/test_system.py 2010-01-24 05:52:01.855000000 +0000 +++ new-tahoe/src/allmydata/test/test_system.py 2010-01-24 05:52:06.340000000 +0000 @@ -17,7 +17,7 @@ from allmydata.interfaces import IDirectoryNode, IFileNode, \ NoSuchChildError, NoSharesError from allmydata.monitor import Monitor -from allmydata.mutable.common import NotMutableError +from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from foolscap.api import DeadReferenceError from twisted.python.failure import Failure @@ -890,11 +890,11 @@ d1.addCallback(lambda res: dirnode.list()) d1.addCallback(self.log, "dirnode.list") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope")) d1.addCallback(self.log, "doing add_file(ro)") ut = upload.Data("I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence="99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut)) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut)) d1.addCallback(self.log, "doing get(ro)") d1.addCallback(lambda res: dirnode.get(u"mydata992")) @@ -902,17 +902,17 @@ self.failUnless(IFileNode.providedBy(filenode))) d1.addCallback(self.log, "doing delete(ro)") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "delete(nope)", None, dirnode.delete, u"mydata992")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "delete(nope)", None, dirnode.delete, u"mydata992")) - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "set_uri(nope)", None, dirnode.set_uri, u"hopeless", self.uri, self.uri)) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "set_uri(nope)", None, dirnode.set_uri, u"hopeless", self.uri, self.uri)) d1.addCallback(lambda res: self.shouldFail2(NoSuchChildError, "get(missing)", "missing", dirnode.get, u"missing")) personal = self._personal_node - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "mv from readonly", None, dirnode.move_child_to, u"mydata992", personal, u"nope")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mv from readonly", None, dirnode.move_child_to, u"mydata992", personal, u"nope")) d1.addCallback(self.log, "doing move_child_to(ro)2") - d1.addCallback(lambda res: self.shouldFail2(NotMutableError, "mv to readonly", None, personal.move_child_to, u"sekrit data", dirnode, u"nope")) + d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mv to readonly", None, personal.move_child_to, u"sekrit data", dirnode, u"nope")) d1.addCallback(self.log, "finished with _got_s2ro") return d1 diff -rN -u old-tahoe/src/allmydata/test/test_uri.py new-tahoe/src/allmydata/test/test_uri.py --- old-tahoe/src/allmydata/test/test_uri.py 2010-01-24 05:52:01.867000000 +0000 +++ new-tahoe/src/allmydata/test/test_uri.py 2010-01-24 05:52:06.351000000 +0000 @@ -3,7 +3,7 @@ from allmydata import uri from allmydata.util import hashutil, base32 from allmydata.interfaces import IURI, IFileURI, IDirnodeURI, IMutableFileURI, \ - IVerifierURI + IVerifierURI, CapConstraintError class Literal(unittest.TestCase): def _help_test(self, data): @@ -22,8 +22,16 @@ self.failIf(IDirnodeURI.providedBy(u2)) self.failUnlessEqual(u2.data, data) self.failUnlessEqual(u2.get_size(), len(data)) - self.failUnless(u.is_readonly()) - self.failIf(u.is_mutable()) + self.failUnless(u2.is_readonly()) + self.failIf(u2.is_mutable()) + + u2i = uri.from_string(u.to_string(), deep_immutable=True) + self.failUnless(IFileURI.providedBy(u2i)) + self.failIf(IDirnodeURI.providedBy(u2i)) + self.failUnlessEqual(u2i.data, data) + self.failUnlessEqual(u2i.get_size(), len(data)) + self.failUnless(u2i.is_readonly()) + self.failIf(u2i.is_mutable()) u3 = u.get_readonly() self.failUnlessIdentical(u, u3) @@ -51,18 +59,36 @@ fileURI = 'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834' chk1 = uri.CHKFileURI.init_from_string(fileURI) chk2 = uri.CHKFileURI.init_from_string(fileURI) + unk = uri.UnknownURI("lafs://from_the_future") self.failIfEqual(lit1, chk1) self.failUnlessEqual(chk1, chk2) self.failIfEqual(chk1, "not actually a URI") # these should be hashable too - s = set([lit1, chk1, chk2]) - self.failUnlessEqual(len(s), 2) # since chk1==chk2 + s = set([lit1, chk1, chk2, unk]) + self.failUnlessEqual(len(s), 3) # since chk1==chk2 def test_is_uri(self): lit1 = uri.LiteralFileURI("some data").to_string() self.failUnless(uri.is_uri(lit1)) self.failIf(uri.is_uri(None)) + def test_is_literal_file_uri(self): + lit1 = uri.LiteralFileURI("some data").to_string() + self.failUnless(uri.is_literal_file_uri(lit1)) + self.failIf(uri.is_literal_file_uri(None)) + self.failIf(uri.is_literal_file_uri("foo")) + self.failIf(uri.is_literal_file_uri("ro.foo")) + self.failIf(uri.is_literal_file_uri("URI:LITfoo")) + self.failUnless(uri.is_literal_file_uri("ro.URI:LIT:foo")) + self.failUnless(uri.is_literal_file_uri("imm.URI:LIT:foo")) + + def test_has_uri_prefix(self): + self.failUnless(uri.has_uri_prefix("URI:foo")) + self.failUnless(uri.has_uri_prefix("ro.URI:foo")) + self.failUnless(uri.has_uri_prefix("imm.URI:foo")) + self.failIf(uri.has_uri_prefix(None)) + self.failIf(uri.has_uri_prefix("foo")) + class CHKFile(unittest.TestCase): def test_pack(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -88,8 +114,7 @@ self.failUnless(IFileURI.providedBy(u)) self.failIf(IDirnodeURI.providedBy(u)) self.failUnlessEqual(u.get_size(), 1234) - self.failUnless(u.is_readonly()) - self.failIf(u.is_mutable()) + u_ro = u.get_readonly() self.failUnlessIdentical(u, u_ro) he = u.to_human_encoding() @@ -109,11 +134,19 @@ self.failUnless(IFileURI.providedBy(u2)) self.failIf(IDirnodeURI.providedBy(u2)) self.failUnlessEqual(u2.get_size(), 1234) - self.failUnless(u2.is_readonly()) - self.failIf(u2.is_mutable()) + + u2i = uri.from_string(u.to_string(), deep_immutable=True) + self.failUnlessEqual(u.to_string(), u2i.to_string()) + u2ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u.to_string()) + self.failUnlessEqual(u.to_string(), u2ro.to_string()) + u2imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u.to_string()) + self.failUnlessEqual(u.to_string(), u2imm.to_string()) v = u.get_verify_cap() self.failUnless(isinstance(v.to_string(), str)) + self.failUnless(v.is_readonly()) + self.failIf(v.is_mutable()) + v2 = uri.from_string(v.to_string()) self.failUnlessEqual(v, v2) he = v.to_human_encoding() @@ -126,6 +159,8 @@ total_shares=10, size=1234) self.failUnless(isinstance(v3.to_string(), str)) + self.failUnless(v3.is_readonly()) + self.failIf(v3.is_mutable()) def test_pack_badly(self): key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -179,13 +214,20 @@ self.failUnlessEqual(readable["UEB_hash"], base32.b2a(hashutil.uri_extension_hash(ext))) -class Invalid(unittest.TestCase): +class Unknown(unittest.TestCase): def test_from_future(self): # any URI type that we don't recognize should be treated as unknown future_uri = "I am a URI from the future. Whatever you do, don't " u = uri.from_string(future_uri) self.failUnless(isinstance(u, uri.UnknownURI)) self.failUnlessEqual(u.to_string(), future_uri) + self.failUnless(u.get_readonly() is None) + self.failUnless(u.get_error() is None) + + u2 = uri.UnknownURI(future_uri, error=CapConstraintError("...")) + self.failUnlessEqual(u.to_string(), future_uri) + self.failUnless(u2.get_readonly() is None) + self.failUnless(isinstance(u2.get_error(), CapConstraintError)) class Constraint(unittest.TestCase): def test_constraint(self): @@ -226,6 +268,13 @@ self.failUnless(IMutableFileURI.providedBy(u2)) self.failIf(IDirnodeURI.providedBy(u2)) + u2i = uri.from_string(u.to_string(), deep_immutable=True) + self.failUnless(isinstance(u2i, uri.UnknownURI), u2i) + u2ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u.to_string()) + self.failUnless(isinstance(u2ro, uri.UnknownURI), u2ro) + u2imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u.to_string()) + self.failUnless(isinstance(u2imm, uri.UnknownURI), u2imm) + u3 = u2.get_readonly() readkey = hashutil.ssk_readkey_hash(writekey) self.failUnlessEqual(u3.fingerprint, fingerprint) @@ -236,6 +285,13 @@ self.failUnless(IMutableFileURI.providedBy(u3)) self.failIf(IDirnodeURI.providedBy(u3)) + u3i = uri.from_string(u3.to_string(), deep_immutable=True) + self.failUnless(isinstance(u3i, uri.UnknownURI), u3i) + u3ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u3.to_string()) + self.failUnlessEqual(u3.to_string(), u3ro.to_string()) + u3imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u3.to_string()) + self.failUnless(isinstance(u3imm, uri.UnknownURI), u3imm) + he = u3.to_human_encoding() u3_h = uri.ReadonlySSKFileURI.init_from_human_encoding(he) self.failUnlessEqual(u3, u3_h) @@ -249,6 +305,13 @@ self.failUnless(IMutableFileURI.providedBy(u4)) self.failIf(IDirnodeURI.providedBy(u4)) + u4i = uri.from_string(u4.to_string(), deep_immutable=True) + self.failUnless(isinstance(u4i, uri.UnknownURI), u4i) + u4ro = uri.from_string(uri.ALLEGED_READONLY_PREFIX + u4.to_string()) + self.failUnlessEqual(u4.to_string(), u4ro.to_string()) + u4imm = uri.from_string(uri.ALLEGED_IMMUTABLE_PREFIX + u4.to_string()) + self.failUnless(isinstance(u4imm, uri.UnknownURI), u4imm) + u4a = uri.from_string(u4.to_string()) self.failUnlessEqual(u4a, u4) self.failUnless("ReadonlySSKFileURI" in str(u4a)) @@ -291,12 +354,19 @@ self.failIf(IFileURI.providedBy(u2)) self.failUnless(IDirnodeURI.providedBy(u2)) + u2i = uri.from_string(u1.to_string(), deep_immutable=True) + self.failUnless(isinstance(u2i, uri.UnknownURI)) + u3 = u2.get_readonly() self.failUnless(u3.is_readonly()) self.failUnless(u3.is_mutable()) self.failUnless(IURI.providedBy(u3)) self.failIf(IFileURI.providedBy(u3)) self.failUnless(IDirnodeURI.providedBy(u3)) + + u3i = uri.from_string(u2.to_string(), deep_immutable=True) + self.failUnless(isinstance(u3i, uri.UnknownURI)) + u3n = u3._filenode_uri self.failUnless(u3n.is_readonly()) self.failUnless(u3n.is_mutable()) @@ -363,10 +433,16 @@ self.failIf(IFileURI.providedBy(u2)) self.failUnless(IDirnodeURI.providedBy(u2)) + u2i = uri.from_string(u1.to_string(), deep_immutable=True) + self.failUnlessEqual(u1.to_string(), u2i.to_string()) + u3 = u2.get_readonly() self.failUnlessEqual(u3.to_string(), u2.to_string()) self.failUnless(str(u3)) + u3i = uri.from_string(u2.to_string(), deep_immutable=True) + self.failUnlessEqual(u2.to_string(), u3i.to_string()) + u2_verifier = u2.get_verify_cap() self.failUnless(isinstance(u2_verifier, uri.ImmutableDirectoryURIVerifier), diff -rN -u old-tahoe/src/allmydata/test/test_web.py new-tahoe/src/allmydata/test/test_web.py --- old-tahoe/src/allmydata/test/test_web.py 2010-01-24 05:52:01.885000000 +0000 +++ new-tahoe/src/allmydata/test/test_web.py 2010-01-24 05:52:06.361000000 +0000 @@ -7,7 +7,7 @@ from twisted.web import client, error, http from twisted.python import failure, log from nevow import rend -from allmydata import interfaces, uri, webish +from allmydata import interfaces, uri, webish, dirnode from allmydata.storage.shares import get_share_file from allmydata.storage_client import StorageFarmBroker from allmydata.immutable import upload, download @@ -18,6 +18,7 @@ from allmydata.scripts.debug import CorruptShareOptions, corrupt_share from allmydata.util import fileutil, base32 from allmydata.util.consumer import download_to_data +from allmydata.util.netstring import split_netstring from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \ create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri from allmydata.interfaces import IMutableFileNode @@ -366,25 +367,101 @@ self.fail("%s was supposed to Error(404), not get '%s'" % (which, res)) + def _dump_res(self, res): + import traceback + s = "%r\n" % (res,) + if hasattr(res, 'tb_frame'): + s += "Traceback:\n%s\n" % (traceback.format_tb(res),) + if hasattr(res, 'value'): + s += "%r\n" % (res.value,) + if hasattr(res.value, 'tb_frame'): + s += "Traceback:\n%s\n" % (res, res.value, traceback.format_tb(res)) + if hasattr(res.value, 'response'): + s += "Response body:\n%s\n" % (res.value.response,) + return s + + def shouldSucceedGET(self, urlpath, followRedirect=False, + expected_statuscode=http.OK, return_response=False, **kwargs): + d = self.GET(urlpath, followRedirect=followRedirect, return_response=True, **kwargs) + def done((res, statuscode, headers)): + if isinstance(res, failure.Failure): + self.fail(("'GET %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it failed with statuscode %s instead.\n" + "%s\nThe response headers were:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, + self._dump_res(res), headers)) + if str(statuscode) != str(expected_statuscode): + self.fail(("'GET %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it succeeded with statuscode %s instead.\n" + "The response headers were:\n%s\n\n" + "The response body was:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, headers, res)) + if return_response: + return (res, statuscode, headers) + else: + return res + d.addBoth(done) + return d + + def shouldSucceedHEAD(self, urlpath, expected_statuscode=http.OK, + return_response=False, **kwargs): + d = self.HEAD(urlpath, return_response=True, **kwargs) + def done((res, statuscode, headers)): + if isinstance(res, failure.Failure): + self.fail(("'HEAD %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it failed with statuscode %s instead.\n" + "%s\nThe response headers were:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, + self._dump_res(res), headers)) + if str(statuscode) != str(expected_statuscode): + self.fail(("'HEAD %s' with kwargs %r was supposed to succeed with statuscode %s, " + "but it succeeded with statuscode %s instead.\n" + "The response headers were:\n%s\n\n" + "The response body was:\n%s") % ( + urlpath, kwargs, expected_statuscode, statuscode, headers, res)) + if return_response: + return (res, statuscode, headers) + else: + return res + d.addBoth(done) + return d + + def shouldSucceed(self, which, expected_statuscode, callable, *args, **kwargs): + d = defer.maybeDeferred(callable, *args, **kwargs) + def done(res): + if isinstance(res, failure.Failure): + self.fail(("%s:\nAn HTTP op with args %r and kwargs %r was supposed to " + "succeed with statuscode %s, but it failed:\n%s") % ( + which, args, kwargs, expected_statuscode, + self._dump_res(res))) + #if str(statuscode) != str(expected_statuscode): + # self.fail(("%s:\nAn HTTP op with args %r and kwargs %r was supposed to " + # "succeed with statuscode %s, but it succeeded with statuscode %s instead.\n" + # "The response body was:\n%s") % ( + # which, args, kwargs, expected_statuscode, statuscode, res)) + return res + d.addBoth(done) + return d + class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase): def test_create(self): pass def test_welcome(self): - d = self.GET("/") + d = self.shouldSucceedGET("/") def _check(res): self.failUnless('Welcome To Tahoe-LAFS' in res, res) self.s.basedir = 'web/test_welcome' fileutil.make_dirs("web/test_welcome") fileutil.make_dirs("web/test_welcome/private") - return self.GET("/") + return self.shouldSucceedGET("/") d.addCallback(_check) return d def test_provisioning(self): - d = self.GET("/provisioning/") + d = self.shouldSucceedGET("/provisioning/") def _check(res): self.failUnless('Tahoe Provisioning Tool' in res) fields = {'filled': True, @@ -400,9 +477,10 @@ "delete_rate": 10, "lease_timer": 7, } - return self.POST("/provisioning/", **fields) - + return self.shouldSucceed("POST_provisioning-1", http.OK, self.POST, + "/provisioning/", **fields) d.addCallback(_check) + def _check2(res): self.failUnless('Tahoe Provisioning Tool' in res) self.failUnless("Share space consumed: 167.01TB" in res) @@ -422,13 +500,17 @@ "delete_rate": 100, "lease_timer": 7, } - return self.POST("/provisioning/", **fields) + return self.shouldSucceed("POST_provisioning-2", http.OK, self.POST, + "/provisioning/", **fields) d.addCallback(_check2) + def _check3(res): self.failUnless("Share space consumed: huge!" in res) fields = {'filled': True} - return self.POST("/provisioning/", **fields) + return self.shouldSucceed("POST_provisioning-3", http.OK, self.POST, + "/provisioning/", **fields) d.addCallback(_check3) + def _check4(res): self.failUnless("Share space consumed:" in res) d.addCallback(_check4) @@ -442,7 +524,7 @@ except: raise unittest.SkipTest("reliability tool requires NumPy") - d = self.GET("/reliability/") + d = self.shouldSucceedGET("/reliability/") def _check(res): self.failUnless('Tahoe Reliability Tool' in res) fields = {'drive_lifetime': "8Y", @@ -471,7 +553,7 @@ mu_num = h.list_all_mapupdate_statuses()[0].get_counter() pub_num = h.list_all_publish_statuses()[0].get_counter() ret_num = h.list_all_retrieve_statuses()[0].get_counter() - d = self.GET("/status", followRedirect=True) + d = self.shouldSucceedGET("/status", followRedirect=True) def _check(res): self.failUnless('Upload and Download Status' in res, res) self.failUnless('"down-%d"' % dl_num in res, res) @@ -480,7 +562,7 @@ self.failUnless('"publish-%d"' % pub_num in res, res) self.failUnless('"retrieve-%d"' % ret_num in res, res) d.addCallback(_check) - d.addCallback(lambda res: self.GET("/status/?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET("/status/?t=json")) def _check_json(res): data = simplejson.loads(res) self.failUnless(isinstance(data, dict)) @@ -489,23 +571,23 @@ # here. d.addCallback(_check_json) - d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/down-%d" % dl_num)) def _check_dl(res): self.failUnless("File Download Status" in res, res) d.addCallback(_check_dl) - d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/up-%d" % ul_num)) def _check_ul(res): self.failUnless("File Upload Status" in res, res) d.addCallback(_check_ul) - d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/mapupdate-%d" % mu_num)) def _check_mapupdate(res): self.failUnless("Mutable File Servermap Update Status" in res, res) d.addCallback(_check_mapupdate) - d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/publish-%d" % pub_num)) def _check_publish(res): self.failUnless("Mutable File Publish Status" in res, res) d.addCallback(_check_publish) - d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num)) + d.addCallback(lambda res: self.shouldSucceedGET("/status/retrieve-%d" % ret_num)) def _check_retrieve(res): self.failUnless("Mutable File Retrieve Status" in res, res) d.addCallback(_check_retrieve) @@ -536,16 +618,15 @@ self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps") def test_GET_FILEURL(self): - d = self.GET(self.public_url + "/foo/bar.txt") + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt") d.addCallback(self.failUnlessIsBarDotTxt) return d def test_GET_FILEURL_range(self): headers = {"range": "bytes=1-10"} - d = self.GET(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): - self.failUnlessEqual(int(status), 206) + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 1-10/%d" % len(self.BAR_CONTENTS)) @@ -556,10 +637,9 @@ def test_GET_FILEURL_partial_range(self): headers = {"range": "bytes=5-"} length = len(self.BAR_CONTENTS) - d = self.GET(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): - self.failUnlessEqual(int(status), 206) + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 5-%d/%d" % (length-1, length)) @@ -569,11 +649,10 @@ def test_HEAD_FILEURL_range(self): headers = {"range": "bytes=1-10"} - d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): + d = self.shouldSucceedHEAD(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnlessEqual(res, "") - self.failUnlessEqual(int(status), 206) self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 1-10/%d" % len(self.BAR_CONTENTS)) @@ -583,10 +662,9 @@ def test_HEAD_FILEURL_partial_range(self): headers = {"range": "bytes=5-"} length = len(self.BAR_CONTENTS) - d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers, - return_response=True) - def _got((res, status, headers)): - self.failUnlessEqual(int(status), 206) + d = self.shouldSucceedHEAD(self.public_url + "/foo/bar.txt", headers=headers, + expected_statuscode=http.PARTIAL_CONTENT, return_response=True) + def _got((res, statuscode, headers)): self.failUnless(headers.has_key("content-range")) self.failUnlessEqual(headers["content-range"][0], "bytes 5-%d/%d" % (length-1, length)) @@ -595,7 +673,7 @@ def test_GET_FILEURL_range_bad(self): headers = {"range": "BOGUS=fizbop-quarnak"} - d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad", + d = self.shouldFail2(error.Error, "GET_FILEURL_range_bad", "400 Bad Request", "Syntactically invalid http range header", self.GET, self.public_url + "/foo/bar.txt", @@ -603,8 +681,9 @@ return d def test_HEAD_FILEURL(self): - d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True) - def _got((res, status, headers)): + d = self.shouldSucceedHEAD(self.public_url + "/foo/bar.txt", + expected_statuscode=http.OK, return_response=True) + def _got((res, statuscode, headers)): self.failUnlessEqual(res, "") self.failUnlessEqual(headers["content-length"][0], str(len(self.BAR_CONTENTS))) @@ -615,27 +694,27 @@ def test_GET_FILEURL_named(self): base = "/file/%s" % urllib.quote(self._bar_txt_uri) base2 = "/named/%s" % urllib.quote(self._bar_txt_uri) - d = self.GET(base + "/@@name=/blah.txt") + d = self.shouldSucceedGET(base + "/@@name=/blah.txt") d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base + "/blah.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base + "/blah.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base + "/ignore/lots/blah.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base2 + "/@@name=/blah.txt")) d.addCallback(self.failUnlessIsBarDotTxt) save_url = base + "?save=true&filename=blah.txt" - d.addCallback(lambda res: self.GET(save_url)) + d.addCallback(lambda res: self.shouldSucceedGET(save_url)) d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t u_fn_e = urllib.quote(u_filename.encode("utf-8")) u_url = base + "?save=true&filename=" + u_fn_e - d.addCallback(lambda res: self.GET(u_url)) + d.addCallback(lambda res: self.shouldSucceedGET(u_url)) d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers return d def test_PUT_FILEURL_named_bad(self): base = "/file/%s" % urllib.quote(self._bar_txt_uri) - d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad", + d = self.shouldFail2(error.Error, "PUT_FILEURL_named_bad", "400 Bad Request", "/file can only be used with GET or HEAD", self.PUT, base + "/@@name=/blah.txt", "") @@ -643,14 +722,14 @@ def test_GET_DIRURL_named_bad(self): base = "/file/%s" % urllib.quote(self._foo_uri) - d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad", + d = self.shouldFail2(error.Error, "PUT_DIRURL_named_bad", "400 Bad Request", "is not a file-cap", self.GET, base + "/@@name=/blah.txt") return d def test_GET_slash_file_bad(self): - d = self.shouldFail2(error.Error, "test_GET_slash_file_bad", + d = self.shouldFail2(error.Error, "GET_slash_file_bad", "404 Not Found", "/file must be followed by a file-cap and a name", self.GET, "/file") @@ -671,7 +750,7 @@ verifier_cap = n.get_verify_cap().to_string() base = "/uri/%s" % urllib.quote(verifier_cap) # client.create_node_from_uri() can't handle verify-caps - d = self.shouldFail2(error.Error, "test_GET_unhandled_URI", + d = self.shouldFail2(error.Error, "GET_unhandled_URI", "400 Bad Request", "GET unknown URI type: can only do t=info", self.GET, base) @@ -679,14 +758,14 @@ def test_GET_FILE_URI(self): base = "/uri/%s" % urllib.quote(self._bar_txt_uri) - d = self.GET(base) + d = self.shouldSucceedGET(base) d.addCallback(self.failUnlessIsBarDotTxt) return d def test_GET_FILE_URI_badchild(self): base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri) errmsg = "Files have no children, certainly not named 'boguschild'" - d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild", + d = self.shouldFail2(error.Error, "GET_FILE_URI_badchild", "400 Bad Request", errmsg, self.GET, base) return d @@ -694,35 +773,42 @@ def test_PUT_FILE_URI_badchild(self): base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri) errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory" - d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild", + d = self.shouldFail2(error.Error, "GET_FILE_URI_badchild", "400 Bad Request", errmsg, self.PUT, base, "") return d + # TODO: version of this with a Unicode filename def test_GET_FILEURL_save(self): - d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true") - # TODO: look at the headers, expect a Content-Disposition: attachment - # header. - d.addCallback(self.failUnlessIsBarDotTxt) + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true", + return_response=True) + def _got((res, statuscode, headers)): + content_disposition = headers["content-disposition"][0] + self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition) + self.failUnlessIsBarDotTxt(res) + d.addCallback(_got) return d def test_GET_FILEURL_missing(self): d = self.GET(self.public_url + "/foo/missing") - d.addBoth(self.should404, "test_GET_FILEURL_missing") + d.addBoth(self.should404, "GET_FILEURL_missing") return d def test_PUT_overwrite_only_files(self): # create a directory, put a file in that directory. contents, n, filecap = self.makefile(8) - d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "") + d = self.shouldSucceed("PUT_overwrite_only_files_1", http.OK, self.PUT, + self.public_url + "/foo/dir?t=mkdir", "") d.addCallback(lambda res: - self.PUT(self.public_url + "/foo/dir/file1.txt", - self.NEWFILE_CONTENTS)) + self.shouldSucceed("PUT_overwrite_only_files_2", http.OK, self.PUT, + self.public_url + "/foo/dir/file1.txt", + self.NEWFILE_CONTENTS)) # try to overwrite the file with replace=only-files # (this should work) d.addCallback(lambda res: - self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files", - filecap)) + self.shouldSucceed("PUT_overwrite_only_files_3", http.OK, self.PUT, + self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files", + filecap)) d.addCallback(lambda res: self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict", "There was already a child by that name, and you asked me " @@ -732,21 +818,19 @@ return d def test_PUT_NEWFILEURL(self): - d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 201) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d = self.shouldSucceed("PUT_NEWFILEURL", http.CREATED, self.PUT, + self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", self.NEWFILE_CONTENTS)) return d def test_PUT_NEWFILEURL_not_mutable(self): - d = self.PUT(self.public_url + "/foo/new.txt?mutable=false", - self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 201) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d = self.shouldSucceed("PUT_NEWFILEURL_not_mutable", http.CREATED, self.PUT, + self.public_url + "/foo/new.txt?mutable=false", + self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", self.NEWFILE_CONTENTS)) @@ -755,7 +839,7 @@ def test_PUT_NEWFILEURL_range_bad(self): headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)} target = self.public_url + "/foo/new.txt" - d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad", + d = self.shouldFail2(error.Error, "PUT_NEWFILEURL_range_bad", "501 Not Implemented", "Content-Range in PUT not yet supported", # (and certainly not for immutable files) @@ -766,17 +850,16 @@ return d def test_PUT_NEWFILEURL_mutable(self): - d = self.PUT(self.public_url + "/foo/new.txt?mutable=true", - self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 201) + d = self.shouldSucceed("PUT_NEWFILEURL_mutable", http.CREATED, self.PUT, + self.public_url + "/foo/new.txt?mutable=true", + self.NEWFILE_CONTENTS) def _check_uri(res): u = uri.from_string_mutable_filenode(res) self.failUnless(u.is_mutable()) self.failIf(u.is_readonly()) return res d.addCallback(_check_uri) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(self._foo_node, u"new.txt", @@ -784,7 +867,7 @@ return d def test_PUT_NEWFILEURL_mutable_toobig(self): - d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig", + d = self.shouldFail2(error.Error, "PUT_NEWFILEURL_mutable_toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.PUT, @@ -793,10 +876,9 @@ return d def test_PUT_NEWFILEURL_replace(self): - d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS) - # TODO: we lose the response code, so we can't check this - #self.failUnlessEqual(responsecode, 200) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt") + d = self.shouldSucceed("PUT_NEWFILEURL_replace", http.OK, self.PUT, + self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"bar.txt", self.NEWFILE_CONTENTS)) @@ -819,9 +901,11 @@ return d def test_PUT_NEWFILEURL_mkdirs(self): - d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS) + d = self.shouldSucceed("PUT_NEWFILEURL_mkdirs", http.OK, self.PUT, + self.public_url + "/foo/newdir/new.txt", + self.NEWFILE_CONTENTS) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt") + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt") d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt")) d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir")) d.addCallback(lambda res: @@ -839,26 +923,27 @@ def test_PUT_NEWFILEURL_emptyname(self): # an empty pathname component (i.e. a double-slash) is disallowed - d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname", + d = self.shouldFail2(error.Error, "PUT_NEWFILEURL_emptyname", "400 Bad Request", "The webapi does not allow empty pathname components", self.PUT, self.public_url + "/foo//new.txt", "") return d def test_DELETE_FILEURL(self): - d = self.DELETE(self.public_url + "/foo/bar.txt") + d = self.shouldSucceed("DELETE_FILEURL", http.OK, self.DELETE, + self.public_url + "/foo/bar.txt") d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) return d def test_DELETE_FILEURL_missing(self): d = self.DELETE(self.public_url + "/foo/missing") - d.addBoth(self.should404, "test_DELETE_FILEURL_missing") + d.addBoth(self.should404, "DELETE_FILEURL_missing") return d def test_DELETE_FILEURL_missing2(self): d = self.DELETE(self.public_url + "/missing/missing") - d.addBoth(self.should404, "test_DELETE_FILEURL_missing2") + d.addBoth(self.should404, "DELETE_FILEURL_missing2") return d def failUnlessHasBarDotTxtMetadata(self, res): @@ -875,7 +960,7 @@ # I can't do "GET /path?json", I have to do "GET /path/t=json" # instead. This may make it tricky to emulate the S3 interface # completely. - d = self.GET(self.public_url + "/foo/bar.txt?t=json") + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=json") def _check1(data): self.failUnlessIsBarJSON(data) self.failUnlessHasBarDotTxtMetadata(data) @@ -885,16 +970,16 @@ def test_GET_FILEURL_json_missing(self): d = self.GET(self.public_url + "/foo/missing?json") - d.addBoth(self.should404, "test_GET_FILEURL_json_missing") + d.addBoth(self.should404, "GET_FILEURL_json_missing") return d def test_GET_FILEURL_uri(self): - d = self.GET(self.public_url + "/foo/bar.txt?t=uri") + d = self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=uri") def _check(res): self.failUnlessEqual(res, self._bar_txt_uri) d.addCallback(_check) d.addCallback(lambda res: - self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri")) + self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=readonly-uri")) def _check2(res): # for now, for files, uris and readonly-uris are the same self.failUnlessEqual(res, self._bar_txt_uri) @@ -910,14 +995,14 @@ def test_GET_FILEURL_uri_missing(self): d = self.GET(self.public_url + "/foo/missing?t=uri") - d.addBoth(self.should404, "test_GET_FILEURL_uri_missing") + d.addBoth(self.should404, "GET_FILEURL_uri_missing") return d def test_GET_DIRURL(self): # the addSlash means we get a redirect here # from /uri/$URI/foo/ , we need ../../../ to get back to the root ROOT = "../../.." - d = self.GET(self.public_url + "/foo", followRedirect=True) + d = self.shouldSucceedGET(self.public_url + "/foo", followRedirect=True) def _check(res): self.failUnless(('Return to Welcome page' % ROOT) in res, res) @@ -954,9 +1039,9 @@ self.failUnless(re.search(get_sub, res), res) d.addCallback(_check) - # look at a directory which is readonly + # look at a readonly directory d.addCallback(lambda res: - self.GET(self.public_url + "/reedownlee", followRedirect=True)) + self.shouldSucceedGET(self.public_url + "/reedownlee", followRedirect=True)) def _check2(res): self.failUnless("(read-only)" in res, res) self.failIf("Upload a file" in res, res) @@ -964,14 +1049,14 @@ # and at a directory that contains a readonly directory d.addCallback(lambda res: - self.GET(self.public_url, followRedirect=True)) + self.shouldSucceedGET(self.public_url, followRedirect=True)) def _check3(res): self.failUnless(re.search('DIR-RO' r'\s+reedownlee', res), res) d.addCallback(_check3) # and an empty directory - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty/")) def _check4(res): self.failUnless("directory is empty" in res, res) MKDIR_BUTTON_RE=re.compile('.*Create a new directory in this directory.*', re.I) @@ -981,7 +1066,7 @@ return d def test_GET_DIRURL_badtype(self): - d = self.shouldHTTPError("test_GET_DIRURL_badtype", + d = self.shouldHTTPError("GET_DIRURL_badtype", 400, "Bad Request", "bad t=bogus", self.GET, @@ -989,14 +1074,14 @@ return d def test_GET_DIRURL_json(self): - d = self.GET(self.public_url + "/foo?t=json") + d = self.shouldSucceedGET(self.public_url + "/foo?t=json") d.addCallback(self.failUnlessIsFooJSON) return d def test_POST_DIRURL_manifest_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_manifest_no_ophandle", + "POST_DIRURL_manifest_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-manifest") @@ -1005,8 +1090,9 @@ def test_POST_DIRURL_manifest(self): d = defer.succeed(None) def getman(ignored, output): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125", - followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_manifest", http.OK, self.POST, + self.public_url + "/foo/?t=start-manifest&ophandle=125", + followRedirect=True) d.addCallback(self.wait_for_operation, "125") d.addCallback(self.get_operation_results, "125", output) return d @@ -1019,7 +1105,7 @@ d.addCallback(_got_html) # both t=status and unadorned GET should be identical - d.addCallback(lambda res: self.GET("/operations/125")) + d.addCallback(lambda res: self.shouldSucceedGET("/operations/125")) d.addCallback(_got_html) d.addCallback(getman, "html") @@ -1047,15 +1133,16 @@ def test_POST_DIRURL_deepsize_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_deepsize_no_ophandle", + "POST_DIRURL_deepsize_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-deep-size") return d def test_POST_DIRURL_deepsize(self): - d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126", - followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_deepsize", http.OK, self.POST, + self.public_url + "/foo/?t=start-deep-size&ophandle=126", + followRedirect=True) d.addCallback(self.wait_for_operation, "126") d.addCallback(self.get_operation_results, "126", "json") def _got_json(data): @@ -1075,15 +1162,16 @@ def test_POST_DIRURL_deepstats_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_deepstats_no_ophandle", + "POST_DIRURL_deepstats_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-deep-stats") return d def test_POST_DIRURL_deepstats(self): - d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127", - followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_deepstats", http.OK, self.POST, + self.public_url + "/foo/?t=start-deep-stats&ophandle=127", + followRedirect=True) d.addCallback(self.wait_for_operation, "127") d.addCallback(self.get_operation_results, "127", "json") def _got_json(stats): @@ -1109,7 +1197,8 @@ return d def test_POST_DIRURL_stream_manifest(self): - d = self.POST(self.public_url + "/foo/?t=stream-manifest") + d = self.shouldSucceed("POST_DIRURL_stream_manifest", http.OK, self.POST, + self.public_url + "/foo/?t=stream-manifest") def _check(res): self.failUnless(res.endswith("\n")) units = [simplejson.loads(t) for t in res[:-1].split("\n")] @@ -1129,21 +1218,22 @@ return d def test_GET_DIRURL_uri(self): - d = self.GET(self.public_url + "/foo?t=uri") + d = self.shouldSucceedGET(self.public_url + "/foo?t=uri") def _check(res): self.failUnlessEqual(res, self._foo_uri) d.addCallback(_check) return d def test_GET_DIRURL_readonly_uri(self): - d = self.GET(self.public_url + "/foo?t=readonly-uri") + d = self.shouldSucceedGET(self.public_url + "/foo?t=readonly-uri") def _check(res): self.failUnlessEqual(res, self._foo_readonly_uri) d.addCallback(_check) return d def test_PUT_NEWDIRURL(self): - d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.shouldSucceed("PUT_NEWDIRURL", http.OK, self.PUT, + self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -1151,7 +1241,8 @@ return d def test_POST_NEWDIRURL(self): - d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.shouldSucceed("POST_NEWDIRURL", http.OK, self.POST2, + self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -1160,30 +1251,41 @@ def test_POST_NEWDIRURL_emptyname(self): # an empty pathname component (i.e. a double-slash) is disallowed - d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname", + d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname", "400 Bad Request", "The webapi does not allow empty pathname components, i.e. a double slash", self.POST, self.public_url + "//?t=mkdir") return d def test_POST_NEWDIRURL_initial_children(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_initial_children() + d = self.shouldSucceed("POST_NEWDIRURL_initial_children", http.OK, self.POST2, + self.public_url + "/foo/newdir?t=mkdir-with-children", + simplejson.dumps(newkids)) def _check(uri): n = self.s.create_node_from_uri(uri.strip()) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable", - filecap2)) + self.failUnlessRWChildURIIs(n, u"child-mutable", + caps['filecap2'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable-ro", - filecap3)) + self.failUnlessROChildURIIs(n, u"child-mutable-ro", + caps['filecap3'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild", dircap)) + self.failUnlessROChildURIIs(n, u"unknownchild-ro", + caps['unknown_rocap'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"unknownchild-rw", + caps['unknown_rwcap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"dirchild", + caps['dircap'])) return d2 d.addCallback(_check) d.addCallback(lambda res: @@ -1191,21 +1293,26 @@ d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) return d def test_POST_NEWDIRURL_immutable(self): - (newkids, filecap1, immdircap) = self._create_immutable_children() - d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_immutable_children() + d = self.shouldSucceed("POST_NEWDIRURL_immutable", http.OK, self.POST2, + self.public_url + "/foo/newdir?t=mkdir-immutable", + simplejson.dumps(newkids)) def _check(uri): n = self.s.create_node_from_uri(uri.strip()) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild-imm", - immdircap)) + self.failUnlessROChildURIIs(n, u"dirchild-imm", + caps['immdircap'])) return d2 d.addCallback(_check) d.addCallback(lambda res: @@ -1213,25 +1320,27 @@ d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap) + d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap']) d.addErrback(self.explain_web_error) return d def test_POST_NEWDIRURL_immutable_bad(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad", + (newkids, caps) = self._create_initial_children() + d = self.shouldFail2(error.Error, "POST_NEWDIRURL_immutable_bad", "400 Bad Request", - "a mkdir-immutable operation was given a child that was not itself immutable", + "needed to be immutable but was not", self.POST2, self.public_url + "/foo/newdir?t=mkdir-immutable", simplejson.dumps(newkids)) return d def test_PUT_NEWDIRURL_exists(self): - d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "") + d = self.shouldSucceed("PUT_NEWDIRURL_exists", http.OK, self.PUT, + self.public_url + "/foo/sub?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"sub")) d.addCallback(lambda res: self._foo_node.get(u"sub")) @@ -1249,18 +1358,21 @@ d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"]) return d - def test_PUT_NEWDIRURL_mkdir_p(self): + def test_POST_NEWDIRURL_mkdir_p(self): d = defer.succeed(None) - d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp')) + d.addCallback(lambda res: self.shouldSucceed("POST_NEWDIRURL_mkdir_p-1", http.OK, self.POST, + self.public_url + "/foo", t='mkdir', name='mkp')) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp")) d.addCallback(lambda res: self._foo_node.get(u"mkp")) def mkdir_p(mkpnode): url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri()) - d = self.POST(url) + d = self.shouldSucceed("POST_NEWDIRURL_mkdir_p-2", http.OK, self.POST, + url) def made_subsub(ssuri): d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2") d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri)) - d = self.POST(url) + d = self.shouldSucceed("POST_NEWDIRURL_mkdir_p-3", http.OK, self.POST, + url) d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri)) return d d.addCallback(made_subsub) @@ -1269,7 +1381,8 @@ return d def test_PUT_NEWDIRURL_mkdirs(self): - d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "") + d = self.shouldSucceed("PUT_NEWDIRURL_mkdirs", http.OK, self.PUT, + self.public_url + "/foo/subdir/newdir?t=mkdir", "") d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: @@ -1280,21 +1393,22 @@ return d def test_DELETE_DIRURL(self): - d = self.DELETE(self.public_url + "/foo") + d = self.shouldSucceed("DELETE_DIRURL", http.OK, self.DELETE, + self.public_url + "/foo") d.addCallback(lambda res: self.failIfNodeHasChild(self.public_root, u"foo")) return d def test_DELETE_DIRURL_missing(self): d = self.DELETE(self.public_url + "/foo/missing") - d.addBoth(self.should404, "test_DELETE_DIRURL_missing") + d.addBoth(self.should404, "DELETE_DIRURL_missing") d.addCallback(lambda res: self.failUnlessNodeHasChild(self.public_root, u"foo")) return d def test_DELETE_DIRURL_missing2(self): d = self.DELETE(self.public_url + "/missing") - d.addBoth(self.should404, "test_DELETE_DIRURL_missing2") + d.addBoth(self.should404, "DELETE_DIRURL_missing2") return d def dump_root(self): @@ -1346,18 +1460,44 @@ d.addCallback(_check) return d - def failUnlessChildURIIs(self, node, name, expected_uri): + def failUnlessRWChildURIIs(self, node, name, expected_uri): + assert isinstance(name, unicode) + d = node.get_child_at_path(name) + def _check(child): + self.failUnless(child.is_unknown() or not child.is_readonly()) + self.failUnlessEqual(child.get_uri(), expected_uri.strip()) + expected_ro_uri = self._make_readonly(expected_uri) + if expected_ro_uri: + self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip()) + d.addCallback(_check) + return d + + def failUnlessROChildURIIs(self, node, name, expected_uri): assert isinstance(name, unicode) d = node.get_child_at_path(name) def _check(child): + self.failUnless(child.is_unknown() or child.is_readonly()) self.failUnlessEqual(child.get_uri(), expected_uri.strip()) d.addCallback(_check) return d - def failUnlessURIMatchesChild(self, got_uri, node, name): + def failUnlessURIMatchesRWChild(self, got_uri, node, name): + assert isinstance(name, unicode) + d = node.get_child_at_path(name) + def _check(child): + self.failUnless(child.is_unknown() or not child.is_readonly()) + self.failUnlessEqual(child.get_uri(), got_uri.strip()) + expected_ro_uri = self._make_readonly(got_uri) + if expected_ro_uri: + self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip()) + d.addCallback(_check) + return d + + def failUnlessURIMatchesROChild(self, got_uri, node, name): assert isinstance(name, unicode) d = node.get_child_at_path(name) def _check(child): + self.failUnless(child.is_unknown() or child.is_readonly()) self.failUnlessEqual(got_uri.strip(), child.get_uri()) d.addCallback(_check) return d @@ -1366,10 +1506,11 @@ self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents) def test_POST_upload(self): - d = self.POST(self.public_url + "/foo", t="upload", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload", http.OK, self.POST, + self.public_url + "/foo", t="upload", + file=("new.txt", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, u"new.txt", self.NEWFILE_CONTENTS)) @@ -1377,15 +1518,16 @@ def test_POST_upload_unicode(self): filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t - d = self.POST(self.public_url + "/foo", t="upload", - file=(filename, self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_unicode", http.OK, self.POST, + self.public_url + "/foo", t="upload", + file=(filename, self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, filename) + d.addCallback(self.failUnlessURIMatchesROChild, fn, filename) d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, filename, self.NEWFILE_CONTENTS)) target_url = self.public_url + "/foo/" + filename.encode("utf-8") - d.addCallback(lambda res: self.GET(target_url)) + d.addCallback(lambda res: self.shouldSucceedGET(target_url)) d.addCallback(lambda contents: self.failUnlessEqual(contents, self.NEWFILE_CONTENTS, contents)) @@ -1393,24 +1535,26 @@ def test_POST_upload_unicode_named(self): filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t - d = self.POST(self.public_url + "/foo", t="upload", - name=filename, - file=("overridden", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_unicode_named", http.OK, self.POST, + self.public_url + "/foo", t="upload", + name=filename, + file=("overridden", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, filename) + d.addCallback(self.failUnlessURIMatchesROChild, fn, filename) d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, filename, self.NEWFILE_CONTENTS)) target_url = self.public_url + "/foo/" + filename.encode("utf-8") - d.addCallback(lambda res: self.GET(target_url)) + d.addCallback(lambda res: self.shouldSucceedGET(target_url)) d.addCallback(lambda contents: self.failUnlessEqual(contents, self.NEWFILE_CONTENTS, contents)) return d def test_POST_upload_no_link(self): - d = self.POST("/uri", t="upload", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_no_link", http.OK, self.POST, + "/uri", t="upload", + file=("new.txt", self.NEWFILE_CONTENTS)) def _check_upload_results(page): # this should be a page which describes the results of the upload # that just finished. @@ -1449,7 +1593,7 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnless(target.startswith(self.webish_url), target) return client.getPage(target, method="GET") - d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results", + d = self.shouldRedirect2("POST_upload_no_link_whendone_results", check, self.POST, "/uri", t="upload", when_done="/uri/%(uri)s", @@ -1459,8 +1603,9 @@ return d def test_POST_upload_no_link_mutable(self): - d = self.POST("/uri", t="upload", mutable="true", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_no_link_mutable", http.OK, self.POST, + "/uri", t="upload", mutable="true", + file=("new.txt", self.NEWFILE_CONTENTS)) def _check(filecap): filecap = filecap.strip() self.failUnless(filecap.startswith("URI:SSK:"), filecap) @@ -1472,11 +1617,11 @@ d.addCallback(_check) def _check2(data): self.failUnlessEqual(data, self.NEWFILE_CONTENTS) - return self.GET("/uri/%s" % urllib.quote(self.filecap)) + return self.shouldSucceedGET("/uri/%s" % urllib.quote(self.filecap)) d.addCallback(_check2) def _check3(data): self.failUnlessEqual(data, self.NEWFILE_CONTENTS) - return self.GET("/file/%s" % urllib.quote(self.filecap)) + return self.shouldSucceedGET("/file/%s" % urllib.quote(self.filecap)) d.addCallback(_check3) def _check4(data): self.failUnlessEqual(data, self.NEWFILE_CONTENTS) @@ -1485,7 +1630,7 @@ def test_POST_upload_no_link_mutable_toobig(self): d = self.shouldFail2(error.Error, - "test_POST_upload_no_link_mutable_toobig", + "POST_upload_no_link_mutable_toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.POST, @@ -1496,10 +1641,11 @@ def test_POST_upload_mutable(self): # this creates a mutable file - d = self.POST(self.public_url + "/foo", t="upload", mutable="true", - file=("new.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_mutable", http.OK, self.POST, + self.public_url + "/foo", t="upload", mutable="true", + file=("new.txt", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(fn, u"new.txt", self.NEWFILE_CONTENTS)) @@ -1515,10 +1661,11 @@ # now upload it again and make sure that the URI doesn't change NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n" d.addCallback(lambda res: - self.POST(self.public_url + "/foo", t="upload", - mutable="true", - file=("new.txt", NEWER_CONTENTS))) - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + self.shouldSucceed("POST_upload_mutable-again", http.OK, self.POST, + self.public_url + "/foo", t="upload", + mutable="true", + file=("new.txt", NEWER_CONTENTS))) + d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(fn, u"new.txt", NEWER_CONTENTS)) @@ -1533,8 +1680,9 @@ # upload a second time, using PUT instead of POST NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n" d.addCallback(lambda res: - self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS)) - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + self.shouldSucceed("POST_upload_mutable-again-with-PUT", http.OK, self.PUT, + self.public_url + "/foo/new.txt", NEW2_CONTENTS)) + d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessMutableChildContentsAre(fn, u"new.txt", NEW2_CONTENTS)) @@ -1543,8 +1691,8 @@ # slightly differently d.addCallback(lambda res: - self.GET(self.public_url + "/foo/", - followRedirect=True)) + self.shouldSucceedGET(self.public_url + "/foo/", + followRedirect=True)) def _check_page(res): # TODO: assert more about the contents self.failUnless("SSK" in res) @@ -1561,8 +1709,8 @@ # look at the JSON form of the enclosing directory d.addCallback(lambda res: - self.GET(self.public_url + "/foo/?t=json", - followRedirect=True)) + self.shouldSucceedGET(self.public_url + "/foo/?t=json", + followRedirect=True)) def _check_page_json(res): parsed = simplejson.loads(res) self.failUnlessEqual(parsed[0], "dirnode") @@ -1580,7 +1728,7 @@ # and the JSON form of the file d.addCallback(lambda res: - self.GET(self.public_url + "/foo/new.txt?t=json")) + self.shouldSucceedGET(self.public_url + "/foo/new.txt?t=json")) def _check_file_json(res): parsed = simplejson.loads(res) self.failUnlessEqual(parsed[0], "filenode") @@ -1592,10 +1740,10 @@ # and look at t=uri and t=readonly-uri d.addCallback(lambda res: - self.GET(self.public_url + "/foo/new.txt?t=uri")) + self.shouldSucceedGET(self.public_url + "/foo/new.txt?t=uri")) d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri)) d.addCallback(lambda res: - self.GET(self.public_url + "/foo/new.txt?t=readonly-uri")) + self.shouldSucceedGET(self.public_url + "/foo/new.txt?t=readonly-uri")) def _check_ro_uri(res): ro_uri = unicode(self._mutable_node.get_readonly().to_string()) self.failUnlessEqual(res, ro_uri) @@ -1603,15 +1751,15 @@ # make sure we can get to it from /uri/URI d.addCallback(lambda res: - self.GET("/uri/%s" % urllib.quote(self._mutable_uri))) + self.shouldSucceedGET("/uri/%s" % urllib.quote(self._mutable_uri))) d.addCallback(lambda res: self.failUnlessEqual(res, NEW2_CONTENTS)) # and that HEAD computes the size correctly d.addCallback(lambda res: - self.HEAD(self.public_url + "/foo/new.txt", - return_response=True)) - def _got_headers((res, status, headers)): + self.shouldSucceedHEAD(self.public_url + "/foo/new.txt", + return_response=True)) + def _got_headers((res, statuscode, headers)): self.failUnlessEqual(res, "") self.failUnlessEqual(headers["content-length"][0], str(len(NEW2_CONTENTS))) @@ -1621,7 +1769,7 @@ # make sure that size errors are displayed correctly for overwrite d.addCallback(lambda res: self.shouldFail2(error.Error, - "test_POST_upload_mutable-toobig", + "POST_upload_mutable-toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.POST, @@ -1636,7 +1784,7 @@ def test_POST_upload_mutable_toobig(self): d = self.shouldFail2(error.Error, - "test_POST_upload_mutable_toobig", + "POST_upload_mutable_toobig", "413 Request Entity Too Large", "SDMF is limited to one segment, and 10001 > 10000", self.POST, @@ -1660,19 +1808,21 @@ return f def test_POST_upload_replace(self): - d = self.POST(self.public_url + "/foo", t="upload", - file=("bar.txt", self.NEWFILE_CONTENTS)) + d = self.shouldSucceed("POST_upload_replace", http.OK, self.POST, + self.public_url + "/foo", t="upload", + file=("bar.txt", self.NEWFILE_CONTENTS)) fn = self._foo_node - d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt") + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, u"bar.txt", self.NEWFILE_CONTENTS)) return d def test_POST_upload_no_replace_ok(self): - d = self.POST(self.public_url + "/foo?replace=false", t="upload", - file=("new.txt", self.NEWFILE_CONTENTS)) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt")) + d = self.shouldSucceed("POST_upload_no_replace_ok", http.OK, self.POST, + self.public_url + "/foo?replace=false", t="upload", + file=("new.txt", self.NEWFILE_CONTENTS)) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/new.txt")) d.addCallback(lambda res: self.failUnlessEqual(res, self.NEWFILE_CONTENTS)) return d @@ -1685,7 +1835,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d @@ -1696,7 +1846,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d @@ -1712,9 +1862,10 @@ def test_POST_upload_named(self): fn = self._foo_node - d = self.POST(self.public_url + "/foo", t="upload", - name="new.txt", file=self.NEWFILE_CONTENTS) - d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt") + d = self.shouldSucceed("POST_upload_named", http.OK, self.POST, + self.public_url + "/foo", t="upload", + name="new.txt", file=self.NEWFILE_CONTENTS) + d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(fn, u"new.txt", self.NEWFILE_CONTENTS)) @@ -1724,7 +1875,7 @@ d = self.POST(self.public_url + "/foo", t="upload", name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS) d.addBoth(self.shouldFail, error.Error, - "test_POST_upload_named_badfilename", + "POST_upload_named_badfilename", "400 Bad Request", "name= may not contain a slash", ) @@ -1738,7 +1889,8 @@ def test_POST_FILEURL_check(self): bar_url = self.public_url + "/foo/bar.txt" - d = self.POST(bar_url, t="check") + d = self.shouldSucceed("POST_FILEURL_check-1", http.OK, self.POST, + bar_url, t="check") def _check(res): self.failUnless("Healthy :" in res) d.addCallback(_check) @@ -1747,13 +1899,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_FILEURL_check", + self.shouldRedirect2("POST_FILEURL_check-2", _check2, self.POST, bar_url, t="check", when_done=redir_url)) d.addCallback(lambda res: - self.POST(bar_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_FILEURL_check-3", http.OK, self.POST, + bar_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res) self.failUnless("Return to file" in res) @@ -1761,7 +1914,8 @@ d.addCallback(_check3) d.addCallback(lambda res: - self.POST(bar_url, t="check", output="JSON")) + self.shouldSucceed("POST_FILEURL_check-4", http.OK, self.POST, + bar_url, t="check", output="JSON")) def _check_json(res): data = simplejson.loads(res) self.failUnless("storage-index" in data) @@ -1772,7 +1926,8 @@ def test_POST_FILEURL_check_and_repair(self): bar_url = self.public_url + "/foo/bar.txt" - d = self.POST(bar_url, t="check", repair="true") + d = self.shouldSucceed("POST_FILEURL_check_and_repair-1", http.OK, self.POST, + bar_url, t="check", repair="true") def _check(res): self.failUnless("Healthy :" in res) d.addCallback(_check) @@ -1781,13 +1936,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_FILEURL_check_and_repair", + self.shouldRedirect2("POST_FILEURL_check_and_repair-2", _check2, self.POST, bar_url, t="check", repair="true", when_done=redir_url)) d.addCallback(lambda res: - self.POST(bar_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_FILEURL_check_and_repair-3", http.OK, self.POST, + bar_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res) self.failUnless("Return to file" in res) @@ -1797,7 +1953,8 @@ def test_POST_DIRURL_check(self): foo_url = self.public_url + "/foo/" - d = self.POST(foo_url, t="check") + d = self.shouldSucceed("POST_DIRURL_check-1", http.OK, self.POST, + foo_url, t="check") def _check(res): self.failUnless("Healthy :" in res, res) d.addCallback(_check) @@ -1806,13 +1963,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_DIRURL_check", + self.shouldRedirect2("POST_DIRURL_check-2", _check2, self.POST, foo_url, t="check", when_done=redir_url)) d.addCallback(lambda res: - self.POST(foo_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_DIRURL_check-3", http.OK, self.POST, + foo_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res, res) self.failUnless("Return to file/directory" in res) @@ -1820,7 +1978,8 @@ d.addCallback(_check3) d.addCallback(lambda res: - self.POST(foo_url, t="check", output="JSON")) + self.shouldSucceed("POST_DIRURL_check-4", http.OK, self.POST, + foo_url, t="check", output="JSON")) def _check_json(res): data = simplejson.loads(res) self.failUnless("storage-index" in data) @@ -1831,7 +1990,8 @@ def test_POST_DIRURL_check_and_repair(self): foo_url = self.public_url + "/foo/" - d = self.POST(foo_url, t="check", repair="true") + d = self.shouldSucceed("POST_DIRURL_check_and_repair-1", http.OK, self.POST, + foo_url, t="check", repair="true") def _check(res): self.failUnless("Healthy :" in res, res) d.addCallback(_check) @@ -1840,13 +2000,14 @@ self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnlessEqual(target, redir_url) d.addCallback(lambda res: - self.shouldRedirect2("test_POST_DIRURL_check_and_repair", + self.shouldRedirect2("POST_DIRURL_check_and_repair-2", _check2, self.POST, foo_url, t="check", repair="true", when_done=redir_url)) d.addCallback(lambda res: - self.POST(foo_url, t="check", return_to=redir_url)) + self.shouldSucceed("POST_DIRURL_check_and_repair-3", http.OK, self.POST, + foo_url, t="check", return_to=redir_url)) def _check3(res): self.failUnless("Healthy :" in res) self.failUnless("Return to file/directory" in res) @@ -1857,7 +2018,7 @@ def wait_for_operation(self, ignored, ophandle): url = "/operations/" + ophandle url += "?t=status&output=JSON" - d = self.GET(url) + d = self.shouldSucceedGET(url) def _got(res): data = simplejson.loads(res) if not data["finished"]: @@ -1873,7 +2034,7 @@ url += "?t=status" if output: url += "&output=" + output - d = self.GET(url) + d = self.shouldSucceedGET(url) def _got(res): if output and output.lower() == "json": return simplejson.loads(res) @@ -1883,7 +2044,7 @@ def test_POST_DIRURL_deepcheck_no_ophandle(self): d = self.shouldFail2(error.Error, - "test_POST_DIRURL_deepcheck_no_ophandle", + "POST_DIRURL_deepcheck_no_ophandle", "400 Bad Request", "slow operation requires ophandle=", self.POST, self.public_url, t="start-deep-check") @@ -1893,7 +2054,7 @@ def _check_redirect(statuscode, target): self.failUnlessEqual(statuscode, str(http.FOUND)) self.failUnless(target.endswith("/operations/123")) - d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect, + d = self.shouldRedirect2("POST_DIRURL_deepcheck", _check_redirect, self.POST, self.public_url, t="start-deep-check", ophandle="123") d.addCallback(self.wait_for_operation, "123") @@ -1909,7 +2070,7 @@ d.addCallback(_check_html) d.addCallback(lambda res: - self.GET("/operations/123/")) + self.shouldSucceedGET("/operations/123/")) d.addCallback(_check_html) # should be the same as without the slash d.addCallback(lambda res: @@ -1920,7 +2081,7 @@ foo_si = self._foo_node.get_storage_index() foo_si_s = base32.b2a(foo_si) d.addCallback(lambda res: - self.GET("/operations/123/%s?output=JSON" % foo_si_s)) + self.shouldSucceedGET("/operations/123/%s?output=JSON" % foo_si_s)) def _check_foo_json(res): data = simplejson.loads(res) self.failUnlessEqual(data["storage-index"], foo_si_s) @@ -1929,8 +2090,9 @@ return d def test_POST_DIRURL_deepcheck_and_repair(self): - d = self.POST(self.public_url, t="start-deep-check", repair="true", - ophandle="124", output="json", followRedirect=True) + d = self.shouldSucceed("POST_DIRURL_deepcheck_and_repair", http.OK, self.POST, + self.public_url, t="start-deep-check", repair="true", + ophandle="124", output="json", followRedirect=True) d.addCallback(self.wait_for_operation, "124") def _check_json(data): self.failUnlessEqual(data["finished"], True) @@ -1971,45 +2133,47 @@ return d def test_POST_mkdir(self): # return value? - d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir") + d = self.shouldSucceed("POST_mkdir", http.OK, self.POST, + self.public_url + "/foo", t="mkdir", name="newdir") d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, []) return d def test_POST_mkdir_initial_children(self): - newkids, filecap1, ign, ign, ign = self._create_initial_children() - d = self.POST2(self.public_url + - "/foo?t=mkdir-with-children&name=newdir", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_initial_children() + d = self.shouldSucceed("POST_mkdir_initial_children", http.OK, self.POST2, + self.public_url + "/foo?t=mkdir-with-children&name=newdir", + simplejson.dumps(newkids)) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) return d def test_POST_mkdir_immutable(self): - (newkids, filecap1, immdircap) = self._create_immutable_children() - d = self.POST2(self.public_url + - "/foo?t=mkdir-immutable&name=newdir", - simplejson.dumps(newkids)) + (newkids, caps) = self._create_immutable_children() + d = self.shouldSucceed("POST_mkdir_immutable", http.OK, self.POST2, + self.public_url + "/foo?t=mkdir-immutable&name=newdir", + simplejson.dumps(newkids)) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessNodeKeysAre, newkids.keys()) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1) + d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1']) d.addCallback(lambda res: self._foo_node.get(u"newdir")) - d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap) + d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap']) return d def test_POST_mkdir_immutable_bad(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad", + (newkids, caps) = self._create_initial_children() + d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad", "400 Bad Request", - "a mkdir-immutable operation was given a child that was not itself immutable", + "needed to be immutable but was not", self.POST2, self.public_url + "/foo?t=mkdir-immutable&name=newdir", @@ -2017,7 +2181,8 @@ return d def test_POST_mkdir_2(self): - d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "") + d = self.shouldSucceed("POST_mkdir_2", http.OK, self.POST, + self.public_url + "/foo/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"newdir")) d.addCallback(lambda res: self._foo_node.get(u"newdir")) @@ -2025,7 +2190,8 @@ return d def test_POST_mkdirs_2(self): - d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "") + d = self.shouldSucceed("POST_mkdirs_2", http.OK, self.POST, + self.public_url + "/foo/bardir/newdir?t=mkdir", "") d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bardir")) d.addCallback(lambda res: self._foo_node.get(u"bardir")) @@ -2034,7 +2200,8 @@ return d def test_POST_mkdir_no_parentdir_noredirect(self): - d = self.POST("/uri?t=mkdir") + d = self.shouldSucceed("POST_mkdir_no_parentdir_noredirect", http.OK, self.POST, + "/uri?t=mkdir") def _after_mkdir(res): uri.DirectoryURI.init_from_string(res) d.addCallback(_after_mkdir) @@ -2049,21 +2216,43 @@ d.addCallback(_check_target) return d + def _make_readonly(self, u): + ro_uri = uri.from_string(u).get_readonly() + if ro_uri is None: + return None + return ro_uri.to_string() + def _create_initial_children(self): contents, n, filecap1 = self.makefile(12) md1 = {"metakey1": "metavalue1"} filecap2 = make_mutable_file_uri() node3 = self.s.create_node_from_uri(make_mutable_file_uri()) filecap3 = node3.get_readonly_uri() + unknown_rwcap = "lafs://from_the_future" + unknown_rocap = "ro.lafs://readonly_from_the_future" + unknown_immcap = "imm.lafs://immutable_from_the_future" node4 = self.s.create_node_from_uri(make_mutable_file_uri()) dircap = DirectoryNode(node4, None, None).get_uri() - newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, - "metadata": md1, }], - u"child-mutable": ["filenode", {"rw_uri": filecap2}], + newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1, + "ro_uri": self._make_readonly(filecap1), + "metadata": md1, }], + u"child-mutable": ["filenode", {"rw_uri": filecap2, + "ro_uri": self._make_readonly(filecap2)}], u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}], - u"dirchild": ["dirnode", {"rw_uri": dircap}], + u"unknownchild-rw": ["unknown", {"rw_uri": unknown_rwcap, + "ro_uri": unknown_rocap}], + u"unknownchild-ro": ["unknown", {"ro_uri": unknown_rocap}], + u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}], + u"dirchild": ["dirnode", {"rw_uri": dircap, + "ro_uri": self._make_readonly(dircap)}], } - return newkids, filecap1, filecap2, filecap3, dircap + return newkids, {'filecap1': filecap1, + 'filecap2': filecap2, + 'filecap3': filecap3, + 'unknown_rwcap': unknown_rwcap, + 'unknown_rocap': unknown_rocap, + 'unknown_immcap': unknown_immcap, + 'dircap': dircap} def _create_immutable_children(self): contents, n, filecap1 = self.makefile(12) @@ -2071,31 +2260,46 @@ tnode = create_chk_filenode("immutable directory contents\n"*10) dnode = DirectoryNode(tnode, None, None) assert not dnode.is_mutable() + unknown_immcap = "imm.lafs://immutable_from_the_future" immdircap = dnode.get_uri() - newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, - "metadata": md1, }], - u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}], + newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, + "metadata": md1, }], + u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}], + u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}], } - return newkids, filecap1, immdircap + return newkids, {'filecap1': filecap1, + 'unknown_immcap': unknown_immcap, + 'immdircap': immdircap} def test_POST_mkdir_no_parentdir_initial_children(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() - d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids)) + (newkids, caps) = self._create_initial_children() + d = self.shouldSucceed("POST_mkdir_no_parentdir_initial_children", http.OK, self.POST2, + "/uri?t=mkdir-with-children", simplejson.dumps(newkids)) def _after_mkdir(res): self.failUnless(res.startswith("URI:DIR"), res) n = self.s.create_node_from_uri(res) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"child-mutable", + caps['filecap2'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"child-mutable-ro", + caps['filecap3'])) + d2.addCallback(lambda ign: + self.failUnlessRWChildURIIs(n, u"unknownchild-rw", + caps['unknown_rwcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable", - filecap2)) + self.failUnlessROChildURIIs(n, u"unknownchild-ro", + caps['unknown_rocap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-mutable-ro", - filecap3)) + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild", dircap)) + self.failUnlessRWChildURIIs(n, u"dirchild", + caps['dircap'])) return d2 d.addCallback(_after_mkdir) return d @@ -2103,8 +2307,7 @@ def test_POST_mkdir_no_parentdir_unexpected_children(self): # the regular /uri?t=mkdir operation is specified to ignore its body. # Only t=mkdir-with-children pays attention to it. - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() + (newkids, caps) = self._create_initial_children() d = self.shouldHTTPError("POST t=mkdir unexpected children", 400, "Bad Request", "t=mkdir does not accept children=, " @@ -2121,28 +2324,32 @@ return d def test_POST_mkdir_no_parentdir_immutable(self): - (newkids, filecap1, immdircap) = self._create_immutable_children() - d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids)) + (newkids, caps) = self._create_immutable_children() + d = self.shouldSucceed("POST_mkdir_no_parentdir_immutable", http.OK, self.POST2, + "/uri?t=mkdir-immutable", simplejson.dumps(newkids)) def _after_mkdir(res): self.failUnless(res.startswith("URI:DIR"), res) n = self.s.create_node_from_uri(res) d2 = self.failUnlessNodeKeysAre(n, newkids.keys()) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"child-imm", filecap1)) + self.failUnlessROChildURIIs(n, u"child-imm", + caps['filecap1'])) d2.addCallback(lambda ign: - self.failUnlessChildURIIs(n, u"dirchild-imm", - immdircap)) + self.failUnlessROChildURIIs(n, u"unknownchild-imm", + caps['unknown_immcap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-imm", + caps['immdircap'])) return d2 d.addCallback(_after_mkdir) return d def test_POST_mkdir_no_parentdir_immutable_bad(self): - (newkids, filecap1, filecap2, filecap3, - dircap) = self._create_initial_children() + (newkids, caps) = self._create_initial_children() d = self.shouldFail2(error.Error, - "test_POST_mkdir_no_parentdir_immutable_bad", + "POST_mkdir_no_parentdir_immutable_bad", "400 Bad Request", - "a mkdir-immutable operation was given a child that was not itself immutable", + "needed to be immutable but was not", self.POST2, "/uri?t=mkdir-immutable", simplejson.dumps(newkids)) @@ -2150,9 +2357,14 @@ def test_welcome_page_mkdir_button(self): # Fetch the welcome page. - d = self.GET("/") + d = self.shouldSucceedGET("/") def _after_get_welcome_page(res): - MKDIR_BUTTON_RE=re.compile('', re.I) + MKDIR_BUTTON_RE = re.compile( + '' + '' + '', + re.I) mo = MKDIR_BUTTON_RE.search(res) formaction = mo.group(1) formt = mo.group(2) @@ -2168,7 +2380,8 @@ return d def test_POST_mkdir_replace(self): # return value? - d = self.POST(self.public_url + "/foo", t="mkdir", name="sub") + d = self.shouldSucceed("POST_mkdir_replace", http.OK, self.POST, + self.public_url + "/foo", t="mkdir", name="sub") d.addCallback(lambda res: self._foo_node.get(u"sub")) d.addCallback(self.failUnlessNodeKeysAre, []) return d @@ -2250,9 +2463,9 @@ d = client.getPage(url, method="POST", postdata=reqbody) def _then(res): - self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1") - self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2") - self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3") + self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1") + self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2") + self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3") d.addCallback(_then) d.addErrback(self.dump_error) @@ -2263,8 +2476,9 @@ def test_POST_put_uri(self): contents, n, newuri = self.makefile(8) - d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt") + d = self.shouldSucceed("POST_put_uri", http.OK, self.POST, + self.public_url + "/foo", t="uri", name="new.txt", uri=newuri) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", contents)) @@ -2272,8 +2486,9 @@ def test_POST_put_uri_replace(self): contents, n, newuri = self.makefile(8) - d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri) - d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt") + d = self.shouldSucceed("POST_put_uri_replace", http.OK, self.POST, + self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri) + d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt") d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"bar.txt", contents)) @@ -2288,7 +2503,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d @@ -2301,12 +2516,13 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) return d def test_POST_delete(self): - d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt") + d = self.shouldSucceed("POST_delete", http.OK, self.POST, + self.public_url + "/foo", t="delete", name="bar.txt") d.addCallback(lambda res: self._foo_node.list()) def _check(children): self.failIf(u"bar.txt" in children) @@ -2314,40 +2530,43 @@ return d def test_POST_rename_file(self): - d = self.POST(self.public_url + "/foo", t="rename", - from_name="bar.txt", to_name='wibble.txt') + d = self.shouldSucceed("POST_rename_file", http.OK, self.POST, + self.public_url + "/foo", t="rename", + from_name="bar.txt", to_name='wibble.txt') d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt")) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/wibble.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/wibble.txt?t=json")) d.addCallback(self.failUnlessIsBarJSON) return d def test_POST_rename_file_redundant(self): - d = self.POST(self.public_url + "/foo", t="rename", - from_name="bar.txt", to_name='bar.txt') + d = self.shouldSucceed("POST_rename_file_redundant", http.OK, self.POST, + self.public_url + "/foo", t="rename", + from_name="bar.txt", to_name='bar.txt') d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt")) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/bar.txt?t=json")) d.addCallback(self.failUnlessIsBarJSON) return d def test_POST_rename_file_replace(self): # rename a file and replace a directory with it - d = self.POST(self.public_url + "/foo", t="rename", - from_name="bar.txt", to_name='empty') + d = self.shouldSucceed("POST_rename_file_replace", http.OK, self.POST, + self.public_url + "/foo", t="rename", + from_name="bar.txt", to_name='empty') d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt")) d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"empty")) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty?t=json")) d.addCallback(self.failUnlessIsBarJSON) return d @@ -2360,7 +2579,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty?t=json")) d.addCallback(self.failUnlessIsEmptyJSON) return d @@ -2373,7 +2592,7 @@ "409 Conflict", "There was already a child by that name, and you asked me " "to not replace it") - d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/foo/empty?t=json")) d.addCallback(self.failUnlessIsEmptyJSON) return d @@ -2386,7 +2605,7 @@ d = self.POST(self.public_url + "/foo", t="rename", from_name="bar.txt", to_name='kirk/spock.txt') d.addBoth(self.shouldFail, error.Error, - "test_POST_rename_file_slash_fail", + "POST_rename_file_slash_fail", "400 Bad Request", "to_name= may not contain a slash", ) @@ -2395,13 +2614,14 @@ return d def test_POST_rename_dir(self): - d = self.POST(self.public_url, t="rename", - from_name="foo", to_name='plunk') + d = self.shouldSucceed("POST_rename_dir", http.OK, self.POST, + self.public_url, t="rename", + from_name="foo", to_name='plunk') d.addCallback(lambda res: self.failIfNodeHasChild(self.public_root, u"foo")) d.addCallback(lambda res: self.failUnlessNodeHasChild(self.public_root, u"plunk")) - d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json")) + d.addCallback(lambda res: self.shouldSucceedGET(self.public_url + "/plunk?t=json")) d.addCallback(self.failUnlessIsFooJSON) return d @@ -2436,24 +2656,24 @@ d.addCallback(lambda res: self.GET(base+"&t=json")) d.addBoth(self.shouldRedirect, targetbase+"?t=json") d.addCallback(self.log, "about to get file by uri") - d.addCallback(lambda res: self.GET(base, followRedirect=True)) + d.addCallback(lambda res: self.shouldSucceedGET(base, followRedirect=True)) d.addCallback(self.failUnlessIsBarDotTxt) d.addCallback(self.log, "got file by uri, about to get dir by uri") - d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri, - followRedirect=True)) + d.addCallback(lambda res: self.shouldSucceedGET("/uri?uri=%s&t=json" % self._foo_uri, + followRedirect=True)) d.addCallback(self.failUnlessIsFooJSON) d.addCallback(self.log, "got dir by uri") return d def test_GET_URI_form_bad(self): - d = self.shouldFail2(error.Error, "test_GET_URI_form_bad", + d = self.shouldFail2(error.Error, "GET_URI_form_bad", "400 Bad Request", "GET /uri requires uri=", self.GET, "/uri") return d def test_GET_rename_form(self): - d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt", + d = self.shouldSucceedGET(self.public_url + "/foo?t=rename-form&name=bar.txt", followRedirect=True) def _check(res): self.failUnless('name="when_done" value="."' in res, res) @@ -2468,23 +2688,23 @@ def test_GET_URI_URL(self): base = "/uri/%s" % self._bar_txt_uri - d = self.GET(base) + d = self.shouldSucceedGET(base) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base+"?filename=bar.txt")) + d.addCallback(lambda res: self.shouldSucceedGET(base+"?filename=bar.txt")) d.addCallback(self.failUnlessIsBarDotTxt) - d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true")) + d.addCallback(lambda res: self.shouldSucceedGET(base+"?filename=bar.txt&save=true")) d.addCallback(self.failUnlessIsBarDotTxt) return d def test_GET_URI_URL_dir(self): base = "/uri/%s?t=json" % self._foo_uri - d = self.GET(base) + d = self.shouldSucceedGET(base) d.addCallback(self.failUnlessIsFooJSON) return d def test_GET_URI_URL_missing(self): base = "/uri/%s" % self._bad_file_uri - d = self.shouldHTTPError("test_GET_URI_URL_missing", + d = self.shouldHTTPError("GET_URI_URL_missing", http.GONE, None, "NotEnoughSharesError", self.GET, base) # TODO: how can we exercise both sides of WebDownloadTarget.fail @@ -2502,9 +2722,9 @@ d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri)) d.addCallback(lambda res: - self.failUnlessChildURIIs(self.public_root, - u"foo", - new_uri)) + self.failUnlessRWChildURIIs(self.public_root, + u"foo", + new_uri)) return d d.addCallback(_made_dir) return d @@ -2515,32 +2735,33 @@ new_uri = dn.get_uri() # replace /foo with a new (empty) directory, but ask that # replace=false, so it should fail - d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace", + d = self.shouldFail2(error.Error, "PUT_DIRURL_uri_noreplace", "409 Conflict", "There was already a child by that name, and you asked me to not replace it", self.PUT, self.public_url + "/foo?t=uri&replace=false", new_uri) d.addCallback(lambda res: - self.failUnlessChildURIIs(self.public_root, - u"foo", - self._foo_uri)) + self.failUnlessRWChildURIIs(self.public_root, + u"foo", + self._foo_uri)) return d d.addCallback(_made_dir) return d def test_PUT_DIRURL_bad_t(self): - d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t", + d = self.shouldFail2(error.Error, "PUT_DIRURL_bad_t", "400 Bad Request", "PUT to a directory", self.PUT, self.public_url + "/foo?t=BOGUS", "") d.addCallback(lambda res: - self.failUnlessChildURIIs(self.public_root, - u"foo", - self._foo_uri)) + self.failUnlessRWChildURIIs(self.public_root, + u"foo", + self._foo_uri)) return d def test_PUT_NEWFILEURL_uri(self): contents, n, new_uri = self.makefile(8) - d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri) + d = self.shouldSucceed("PUT_NEWFILEURL_uri", http.OK, self.PUT, + self.public_url + "/foo/new.txt?t=uri", new_uri) d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri)) d.addCallback(lambda res: self.failUnlessChildContentsAre(self._foo_node, u"new.txt", @@ -2567,13 +2788,14 @@ def test_PUT_NEWFILE_URI(self): file_contents = "New file contents here\n" - d = self.PUT("/uri", file_contents) + d = self.shouldSucceed("PUT_NEWFILE_URI", http.OK, self.PUT, + "/uri", file_contents) def _check(uri): assert isinstance(uri, str), uri self.failUnless(uri in FakeCHKFileNode.all_contents) self.failUnlessEqual(FakeCHKFileNode.all_contents[uri], file_contents) - return self.GET("/uri/%s" % uri) + return self.shouldSucceedGET("/uri/%s" % uri) d.addCallback(_check) def _check2(res): self.failUnlessEqual(res, file_contents) @@ -2582,13 +2804,14 @@ def test_PUT_NEWFILE_URI_not_mutable(self): file_contents = "New file contents here\n" - d = self.PUT("/uri?mutable=false", file_contents) + d = self.shouldSucceed("PUT_NEWFILE_URI_not_mutable", http.OK, self.PUT, + "/uri?mutable=false", file_contents) def _check(uri): assert isinstance(uri, str), uri self.failUnless(uri in FakeCHKFileNode.all_contents) self.failUnlessEqual(FakeCHKFileNode.all_contents[uri], file_contents) - return self.GET("/uri/%s" % uri) + return self.shouldSucceedGET("/uri/%s" % uri) d.addCallback(_check) def _check2(res): self.failUnlessEqual(res, file_contents) @@ -2605,7 +2828,8 @@ def test_PUT_NEWFILE_URI_mutable(self): file_contents = "New file contents here\n" - d = self.PUT("/uri?mutable=true", file_contents) + d = self.shouldSucceed("PUT_NEWFILE_URI_mutable", http.OK, self.PUT, + "/uri?mutable=true", file_contents) def _check1(filecap): filecap = filecap.strip() self.failUnless(filecap.startswith("URI:SSK:"), filecap) @@ -2617,7 +2841,7 @@ d.addCallback(_check1) def _check2(data): self.failUnlessEqual(data, file_contents) - return self.GET("/uri/%s" % urllib.quote(self.filecap)) + return self.shouldSucceedGET("/uri/%s" % urllib.quote(self.filecap)) d.addCallback(_check2) def _check3(res): self.failUnlessEqual(res, file_contents) @@ -2625,19 +2849,21 @@ return d def test_PUT_mkdir(self): - d = self.PUT("/uri?t=mkdir", "") + d = self.shouldSucceed("PUT_mkdir", http.OK, self.PUT, + "/uri?t=mkdir", "") def _check(uri): n = self.s.create_node_from_uri(uri.strip()) d2 = self.failUnlessNodeKeysAre(n, []) d2.addCallback(lambda res: - self.GET("/uri/%s?t=json" % uri)) + self.shouldSucceedGET("/uri/%s?t=json" % uri)) return d2 d.addCallback(_check) d.addCallback(self.failUnlessIsEmptyJSON) return d def test_POST_check(self): - d = self.POST(self.public_url + "/foo", t="check", name="bar.txt") + d = self.shouldSucceed("POST_check", http.OK, self.POST, + self.public_url + "/foo", t="check", name="bar.txt") def _done(res): # this returns a string form of the results, which are probably # None since we're using fake filenodes. @@ -2650,7 +2876,7 @@ def test_bad_method(self): url = self.webish_url + self.public_url + "/foo/bar.txt" - d = self.shouldHTTPError("test_bad_method", + d = self.shouldHTTPError("bad_method", 501, "Not Implemented", "I don't know how to treat a BOGUS request.", client.getPage, url, method="BOGUS") @@ -2658,28 +2884,30 @@ def test_short_url(self): url = self.webish_url + "/uri" - d = self.shouldHTTPError("test_short_url", 501, "Not Implemented", + d = self.shouldHTTPError("short_url", 501, "Not Implemented", "I don't know how to treat a DELETE request.", client.getPage, url, method="DELETE") return d def test_ophandle_bad(self): url = self.webish_url + "/operations/bogus?t=status" - d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found", + d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found", "unknown/expired handle 'bogus'", client.getPage, url) return d def test_ophandle_cancel(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128", - followRedirect=True) + d = self.shouldSucceed("ophandle_cancel-1", http.OK, self.POST, + self.public_url + "/foo/?t=start-manifest&ophandle=128", + followRedirect=True) d.addCallback(lambda ignored: - self.GET("/operations/128?t=status&output=JSON")) + self.shouldSucceedGET("/operations/128?t=status&output=JSON")) def _check1(res): data = simplejson.loads(res) self.failUnless("finished" in data, res) monitor = self.ws.root.child_operations.handles["128"][0] - d = self.POST("/operations/128?t=cancel&output=JSON") + d = self.shouldSucceed("ophandle_cancel-2", http.OK, self.POST, + "/operations/128?t=cancel&output=JSON") def _check2(res): data = simplejson.loads(res) self.failUnless("finished" in data, res) @@ -2689,7 +2917,7 @@ return d d.addCallback(_check1) d.addCallback(lambda ignored: - self.shouldHTTPError("test_ophandle_cancel", + self.shouldHTTPError("ophandle_cancel", 404, "404 Not Found", "unknown/expired handle '128'", self.GET, @@ -2700,7 +2928,7 @@ d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60", followRedirect=True) d.addCallback(lambda ignored: - self.GET("/operations/129?t=status&output=JSON&retain-for=0")) + self.shouldSucceedGET("/operations/129?t=status&output=JSON&retain-for=0")) def _check1(res): data = simplejson.loads(res) self.failUnless("finished" in data, res) @@ -2708,7 +2936,7 @@ # the retain-for=0 will cause the handle to be expired very soon d.addCallback(self.stall, 2.0) d.addCallback(lambda ignored: - self.shouldHTTPError("test_ophandle_retainfor", + self.shouldHTTPError("ophandle_retainfor", 404, "404 Not Found", "unknown/expired handle '129'", self.GET, @@ -2716,14 +2944,15 @@ return d def test_ophandle_release_after_complete(self): - d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130", - followRedirect=True) + d = self.shouldSucceed("ophandle_release_after_complete", http.OK, self.POST, + self.public_url + "/foo/?t=start-manifest&ophandle=130", + followRedirect=True) d.addCallback(self.wait_for_operation, "130") d.addCallback(lambda ignored: - self.GET("/operations/130?t=status&output=JSON&release-after-complete=true")) + self.shouldSucceedGET("/operations/130?t=status&output=JSON&release-after-complete=true")) # the release-after-complete=true will cause the handle to be expired d.addCallback(lambda ignored: - self.shouldHTTPError("test_ophandle_release_after_complete", + self.shouldHTTPError("ophandle_release_after_complete", 404, "404 Not Found", "unknown/expired handle '130'", self.GET, @@ -2731,7 +2960,8 @@ return d def test_incident(self): - d = self.POST("/report_incident", details="eek") + d = self.shouldSucceed("incident", http.OK, self.POST, + "/report_incident", details="eek") def _done(res): self.failUnless("Thank you for your report!" in res, res) d.addCallback(_done) @@ -2744,7 +2974,7 @@ f.write("hello") f.close() - d = self.GET("/static/subdir/hello.txt") + d = self.shouldSucceedGET("/static/subdir/hello.txt") def _check(res): self.failUnlessEqual(res, "hello") d.addCallback(_check) @@ -2757,7 +2987,7 @@ self.failUnlessEqual(common.parse_replace_arg("false"), False) self.failUnlessEqual(common.parse_replace_arg("only-files"), "only-files") - self.shouldFail(AssertionError, "test_parse_replace_arg", "", + self.shouldFail(AssertionError, "parse_replace_arg", "", common.parse_replace_arg, "only_fles") def test_abbreviate_time(self): @@ -3062,71 +3292,246 @@ d.addErrback(self.explain_web_error) return d - def test_unknown(self): + def test_unknown(self, immutable=False): self.basedir = "web/Grid/unknown" + if immutable: + self.basedir = "web/Grid/unknown-immutable" + self.set_up_grid() c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." # the future cap format may contain slashes, which must be tolerated - expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap, + expected_info_url = "uri/%s?t=info" % urllib.quote(future_write_uri, safe="") - future_node = UnknownNode(future_writecap, future_readcap) - d = c0.create_dirnode() + if immutable: + name = u"future-imm" + future_node = UnknownNode(None, future_read_uri, deep_immutable=True) + d = c0.create_immutable_dirnode({name: (future_node, {})}) + else: + name = u"future" + future_node = UnknownNode(future_write_uri, future_read_uri) + d = c0.create_dirnode() + def _stash_root_and_create_file(n): self.rootnode = n self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/" self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/" - return self.rootnode.set_node(u"future", future_node) + if not immutable: + return self.rootnode.set_node(name, future_node) d.addCallback(_stash_root_and_create_file) + # make sure directory listing tolerates unknown nodes d.addCallback(lambda ign: self.GET(self.rooturl)) - def _check_html(res): - self.failUnlessIn("future", res) - # find the More Info link for "future", should be relative + def _check_directory_html(res): + self.failUnlessIn("%s" % (str(name),), res) + # find the More Info link for name, should be relative mo = re.search(r'More Info', res) info_url = mo.group(1) - self.failUnlessEqual(info_url, "future?t=info") + self.failUnlessEqual(info_url, "%s?t=info" % (str(name),)) + d.addCallback(_check_directory_html) - d.addCallback(_check_html) d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) - def _check_json(res, expect_writecap): + def _check_directory_json(res, expect_rw_uri): data = simplejson.loads(res) self.failUnlessEqual(data[0], "dirnode") - f = data[1]["children"]["future"] + f = data[1]["children"][name] self.failUnlessEqual(f[0], "unknown") - if expect_writecap: - self.failUnlessEqual(f[1]["rw_uri"], future_writecap) + if expect_rw_uri: + self.failUnlessEqual(f[1]["rw_uri"], future_write_uri) else: self.failIfIn("rw_uri", f[1]) - self.failUnlessEqual(f[1]["ro_uri"], future_readcap) + self.failUnlessEqual(f[1]["ro_uri"], + ("imm." if immutable else "ro.") + future_read_uri) self.failUnless("metadata" in f[1]) - d.addCallback(_check_json, expect_writecap=True) - d.addCallback(lambda ign: self.GET(expected_info_url)) - def _check_info(res, expect_readcap): + d.addCallback(_check_directory_json, expect_rw_uri=not immutable) + + def _check_info(res, expect_rw_uri, expect_ro_uri): self.failUnlessIn("Object Type: unknown", res) - self.failUnlessIn(future_writecap, res) - if expect_readcap: - self.failUnlessIn(future_readcap, res) + if expect_rw_uri: + self.failUnlessIn(future_write_uri, res) + if expect_ro_uri: + self.failUnlessIn(future_read_uri, res) + else: + self.failIfIn(future_read_uri, res) self.failIfIn("Raw data as", res) self.failIfIn("Directory writecap", res) self.failIfIn("Checker Operations", res) self.failIfIn("Mutable File Operations", res) self.failIfIn("Directory Operations", res) - d.addCallback(_check_info, expect_readcap=False) - d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info")) - d.addCallback(_check_info, expect_readcap=True) + + # FIXME: these should have expect_rw_uri=not immutable; I don't know + # why they fail. Possibly related to ticket #922. + + d.addCallback(lambda ign: self.GET(expected_info_url)) + d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False) + d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name)))) + d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True) + + def _check_json(res, expect_rw_uri): + data = simplejson.loads(res) + self.failUnlessEqual(data[0], "unknown") + if expect_rw_uri: + self.failUnlessEqual(data[1]["rw_uri"], future_write_uri) + else: + self.failIfIn("rw_uri", data[1]) + self.failUnlessEqual(data[1]["ro_uri"], + ("imm." if immutable else "ro.") + future_read_uri) + # TODO: check metadata contents + self.failUnless("metadata" in data[1]) + + d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name)))) + d.addCallback(_check_json, expect_rw_uri=not immutable) # and make sure that a read-only version of the directory can be - # rendered too. This version will not have future_writecap + # rendered too. This version will not have future_write_uri, whether + # or not future_node was immutable. d.addCallback(lambda ign: self.GET(self.rourl)) - d.addCallback(_check_html) + d.addCallback(_check_directory_html) d.addCallback(lambda ign: self.GET(self.rourl+"?t=json")) - d.addCallback(_check_json, expect_writecap=False) + d.addCallback(_check_directory_json, expect_rw_uri=False) + + d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name)))) + d.addCallback(_check_json, expect_rw_uri=False) + + # TODO: check that getting t=info from the Info link in the ro directory + # works, and does not include the writecap URI. + return d + + def test_immutable_unknown(self): + return self.test_unknown(immutable=True) + + def test_mutant_dirnodes_are_omitted(self): + self.basedir = "web/Grid/mutant_dirnodes_are_omitted" + + self.set_up_grid() + c = self.g.clients[0] + nm = c.nodemaker + self.uris = {} + self.fileurls = {} + + lonely_uri = "URI:LIT:n5xgk" # LIT for "one" + mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + + # This method tests mainly dirnode, but we'd have to duplicate code in order to + # test the dirnode and web layers separately. + + # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap, + # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field. + # When the directory is read, the mutants should be silently disposed of, leaving + # their lonely sibling. + # We don't test the case of a retrieving a cap from the encrypted rw_uri field, + # because immutable directories don't have a writecap and therefore that field + # isn't (and can't be) decrypted. + # TODO: The field still exists in the netstring. Technically we should check what + # happens if something is put there (it should be ignored), but that can wait. + + lonely_child = nm.create_from_cap(lonely_uri) + mutant_ro_child = nm.create_from_cap(mut_read_uri) + mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri) + + def _by_hook_or_by_crook(): + return True + for n in [mutant_ro_child, mutant_write_in_ro_child]: + n.is_allowed_in_immutable_directory = _by_hook_or_by_crook + + mutant_write_in_ro_child.get_write_uri = lambda: None + mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri + + kids = {u"lonely": (lonely_child, {}), + u"ro": (mutant_ro_child, {}), + u"write-in-ro": (mutant_write_in_ro_child, {}), + } + d = c.create_immutable_dirnode(kids) + + def _created(dn): + self.failUnless(isinstance(dn, dirnode.DirectoryNode)) + self.failIf(dn.is_mutable()) + self.failUnless(dn.is_readonly()) + # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail. + self.failIf(hasattr(dn._node, 'get_writekey')) + rep = str(dn) + self.failUnless("RO-IMM" in rep) + cap = dn.get_cap() + self.failUnlessIn("CHK", cap.to_string()) + self.cap = cap + self.rootnode = dn + self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/" + return download_to_data(dn._node) + d.addCallback(_created) + + def _check_data(data): + # Decode the netstring representation of the directory to check that all children + # are present. This is a bit of an abstraction violation, but there's not really + # any other way to do it given that the real DirectoryNode._unpack_contents would + # strip the mutant children out (which is what we're trying to test, later). + position = 0 + numkids = 0 + while position < len(data): + entries, position = split_netstring(data, 1, position) + entry = entries[0] + (name, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) + name = name.decode("utf-8") + self.failUnless(rwcapdata == "") + ro_uri = ro_uri.strip() + if name in kids: + self.failIfEqual(ro_uri, "") + (expected_child, ign) = kids[name] + self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri()) + numkids += 1 + + self.failUnlessEqual(numkids, 3) + return self.rootnode.list() + d.addCallback(_check_data) + + # Now when we use the real directory listing code, the mutants should be absent. + def _check_kids(children): + self.failUnlessEqual(sorted(children.keys()), [u"lonely"]) + lonely_node, lonely_metadata = children[u"lonely"] + + self.failUnlessEqual(lonely_node.get_write_uri(), None) + self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri) + d.addCallback(_check_kids) + + d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) + d.addCallback(lambda n: n.list()) + d.addCallback(_check_kids) # again with dirnode recreated from cap + + # Make sure the lonely child can be listed in HTML... + d.addCallback(lambda ign: self.GET(self.rooturl)) + def _check_html(res): + self.failIfIn("URI:SSK", res) + get_lonely = "".join([r'FILE', + r'\s+', + r'lonely' % (urllib.quote(lonely_uri),), + r'', + r'\s+%d' % len("one"), + ]) + self.failUnless(re.search(get_lonely, res), res) + + # find the More Info link for name, should be relative + mo = re.search(r'More Info', res) + info_url = mo.group(1) + self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url) + d.addCallback(_check_html) + + # ... and in JSON. + d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) + def _check_json(res): + data = simplejson.loads(res) + self.failUnlessEqual(data[0], "dirnode") + listed_children = data[1]["children"] + self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"]) + ll_type, ll_data = listed_children[u"lonely"] + self.failUnlessEqual(ll_type, "filenode") + self.failIf("rw_uri" in ll_data) + self.failUnlessEqual(ll_data["ro_uri"], lonely_uri) + d.addCallback(_check_json) return d def test_deep_check(self): @@ -3159,10 +3564,10 @@ # this tests that deep-check and stream-manifest will ignore # UnknownNode instances. Hopefully this will also cover deep-stats. - future_writecap = "x-tahoe-crazy://I_am_from_the_future." - future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future." - future_node = UnknownNode(future_writecap, future_readcap) - d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node)) + future_write_uri = "x-tahoe-crazy://I_am_from_the_future." + future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." + future_node = UnknownNode(future_write_uri, future_read_uri) + d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node)) def _clobber_shares(ignored): self.delete_shares_numbered(self.uris["sick"], [0,1]) diff -rN -u old-tahoe/src/allmydata/unknown.py new-tahoe/src/allmydata/unknown.py --- old-tahoe/src/allmydata/unknown.py 2010-01-24 05:52:01.896000000 +0000 +++ new-tahoe/src/allmydata/unknown.py 2010-01-24 05:52:06.373000000 +0000 @@ -1,29 +1,146 @@ + from zope.interface import implements from twisted.internet import defer -from allmydata.interfaces import IFilesystemNode +from allmydata.interfaces import IFilesystemNode, MustNotBeUnknownRWError +from allmydata import uri +from allmydata.uri import ALLEGED_READONLY_PREFIX, ALLEGED_IMMUTABLE_PREFIX + + +# See ticket #833 for design rationale of UnknownNodes. + +"""Strip prefixes when storing an URI in a ro_uri field.""" +def strip_prefix_for_ro(ro_uri, deep_immutable): + # It is possible for an alleged-immutable URI to be put into a + # mutable directory. In that case the ALLEGED_IMMUTABLE_PREFIX + # should not be stripped. In other cases, the prefix can safely + # be stripped because it is implied by the context. + + if ro_uri.startswith(ALLEGED_IMMUTABLE_PREFIX): + if not deep_immutable: + return ro_uri + return ro_uri[len(ALLEGED_IMMUTABLE_PREFIX):] + elif ro_uri.startswith(ALLEGED_READONLY_PREFIX): + return ro_uri[len(ALLEGED_READONLY_PREFIX):] + else: + return ro_uri class UnknownNode: implements(IFilesystemNode) - def __init__(self, writecap, readcap): - assert writecap is None or isinstance(writecap, str) - self.writecap = writecap - assert readcap is None or isinstance(readcap, str) - self.readcap = readcap + + def __init__(self, rw_uri, ro_uri, deep_immutable=False, + name=u""): + #traceback.print_stack() + #print '%r.__init__(%r, %r, deep_immutable=%r, name=%r)' % (self, rw_uri, ro_uri, deep_immutable, name) + assert rw_uri is None or isinstance(rw_uri, str) + assert ro_uri is None or isinstance(ro_uri, str) + + # We don't raise errors when creating an UnknownNode; we instead create an + # opaque node that records the error. This avoids breaking operations that + # never store the opaque node. + # Note that this means that if a stored dirnode has only a rw_uri, it + # might be dropped. Any future "write-only" cap formats should have a dummy + # unusable read cap to stop that from happening. + + self.error = None + self.rw_uri = self.ro_uri = None + if rw_uri is not None: + if deep_immutable: + self.error = MustNotBeUnknownRWError("cannot attach unknown rw cap as immutable child", + name, True) + return + elif ro_uri is None: + # If we have a single unknown cap (specified as a single cap + # argument, or from a rw_uri slot when ro_uri has been omitted), + # then we cannot tell whether it is a rw_uri, and we cannot + # diminish it to a ro_uri. Prefixing it with ALLEGED_READONLY_PREFIX + # would not be sufficient because we have no reason to believe + # that it is a ro_uri, so that might grant excess authority. + self.error = MustNotBeUnknownRWError("cannot attach unknown rw cap as child", + name, False) + return + + # If ro_uri definitely fails the constraint, it should be treated as opaque. + if ro_uri is not None: + read_cap = uri.from_string(ro_uri, deep_immutable=deep_immutable, name=name) + if isinstance(read_cap, uri.UnknownURI): + self.error = read_cap.get_error() + if self.error: + return + + if deep_immutable: + # strengthen ro_uri to have ALLEGED_IMMUTABLE_PREFIX + if ro_uri is not None: + if ro_uri.startswith(ALLEGED_IMMUTABLE_PREFIX): + self.ro_uri = ro_uri + elif ro_uri.startswith(ALLEGED_READONLY_PREFIX): + self.ro_uri = ALLEGED_IMMUTABLE_PREFIX + ro_uri[len(ALLEGED_READONLY_PREFIX):] + else: + self.ro_uri = ALLEGED_IMMUTABLE_PREFIX + ro_uri + else: + self.rw_uri = rw_uri + # strengthen ro_uri to have ALLEGED_READONLY_PREFIX + if ro_uri is not None: + if (ro_uri.startswith(ALLEGED_READONLY_PREFIX) or + ro_uri.startswith(ALLEGED_IMMUTABLE_PREFIX)): + self.ro_uri = ro_uri + else: + self.ro_uri = ALLEGED_READONLY_PREFIX + ro_uri + + #print 'self.(error, rw_uri, ro_uri) = (%r, %r, %r)' % (self.error, self.rw_uri, self.ro_uri) + + def get_cap(self): + return uri.UnknownURI(self.rw_uri or self.ro_uri) + + def get_readcap(self): + return uri.UnknownURI(self.ro_uri) + + def is_readonly(self): + raise AssertionError("an UnknownNode might be either read-only or " + "read/write, so we shouldn't be calling is_readonly") + + def is_mutable(self): + raise AssertionError("an UnknownNode might be either mutable or immutable, " + "so we shouldn't be calling is_mutable") + + def is_unknown(self): + return True + + def is_allowed_in_immutable_directory(self): + # An UnknownNode consisting only of a ro_uri is allowed in an + # immutable directory, even though we do not know that it is + # immutable (or even read-only), provided that no error was detected. + return not self.error and not self.rw_uri + + def raise_error(self): + if self.error is not None: + raise self.error + def get_uri(self): - return self.writecap + return self.rw_uri or self.ro_uri + + def get_write_uri(self): + return self.rw_uri + def get_readonly_uri(self): - return self.readcap + return self.ro_uri + def get_storage_index(self): return None + def get_verify_cap(self): return None + def get_repair_cap(self): return None + def get_size(self): return None + def get_current_size(self): return defer.succeed(None) + def check(self, monitor, verify, add_lease): return defer.succeed(None) + def check_and_repair(self, monitor, verify, add_lease): return defer.succeed(None) diff -rN -u old-tahoe/src/allmydata/uri.py new-tahoe/src/allmydata/uri.py --- old-tahoe/src/allmydata/uri.py 2010-01-24 05:52:01.901000000 +0000 +++ new-tahoe/src/allmydata/uri.py 2010-01-24 05:52:06.378000000 +0000 @@ -5,14 +5,16 @@ from allmydata.storage.server import si_a2b, si_b2a from allmydata.util import base32, hashutil from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IImmutableFileURI, \ - IVerifierURI, IMutableFileURI, IDirectoryURI, IReadonlyDirectoryURI + IVerifierURI, IMutableFileURI, IDirectoryURI, IReadonlyDirectoryURI, \ + MustBeDeepImmutableError, MustBeReadonlyError class BadURIError(Exception): pass -# the URI shall be an ascii representation of the file. It shall contain -# enough information to retrieve and validate the contents. It shall be -# expressed in a limited character set (namely [TODO]). +# The URI shall be an ASCII representation of a reference to the file/directory. +# It shall contain enough information to retrieve and validate the contents. +# It shall be expressed in a limited character set (currently base32 plus ':' and +# capital letters, but future URIs might use a larger charset). BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) @@ -39,6 +41,10 @@ return self.to_string() != them.to_string() else: return True + + def is_unknown(self): + return False + def to_human_encoding(self): return 'http://127.0.0.1:3456/uri/'+self.to_string() @@ -97,8 +103,10 @@ def is_readonly(self): return True + def is_mutable(self): return False + def get_readonly(self): return self @@ -157,6 +165,18 @@ self.total_shares, self.size)) + def is_readonly(self): + return True + + def is_mutable(self): + return False + + def get_readonly(self): + return self + + def get_verify_cap(self): + return self + class LiteralFileURI(_BaseURI): implements(IURI, IImmutableFileURI) @@ -297,10 +317,13 @@ def is_readonly(self): return True + def is_mutable(self): return True + def get_readonly(self): return self + def get_verify_cap(self): return SSKVerifierURI(self.storage_index, self.fingerprint) @@ -334,6 +357,15 @@ return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), base32.b2a(self.fingerprint)) + def is_readonly(self): + return True + def is_mutable(self): + return False + def get_readonly(self): + return self + def get_verify_cap(self): + return self + class _DirectoryBaseURI(_BaseURI): implements(IURI, IDirnodeURI) def __init__(self, filenode_uri=None): @@ -376,12 +408,12 @@ def abbrev_si(self): return base32.b2a(self._filenode_uri.storage_index)[:5] - def get_filenode_cap(self): - return self._filenode_uri - def is_mutable(self): return True + def get_filenode_cap(self): + return self._filenode_uri + def get_verify_cap(self): return DirectoryURIVerifier(self._filenode_uri.get_verify_cap()) @@ -432,12 +464,12 @@ assert isinstance(filenode_uri, self.INNER_URI_CLASS), filenode_uri _DirectoryBaseURI.__init__(self, filenode_uri) - def is_mutable(self): - return False - def is_readonly(self): return True + def is_mutable(self): + return False + def get_readonly(self): return self @@ -460,6 +492,7 @@ # LIT caps have no verifier, since they aren't distributed return None + def wrap_dirnode_cap(filecap): if isinstance(filecap, WriteableSSKFileURI): return DirectoryURI(filecap) @@ -469,7 +502,8 @@ return ImmutableDirectoryURI(filecap) if isinstance(filecap, LiteralFileURI): return LiteralDirectoryURI(filecap) - assert False, "cannot wrap a dirnode around %s" % filecap.__class__ + assert False, "cannot interpret as a directory cap: %s" % filecap.__class__ + class DirectoryURIVerifier(_DirectoryBaseURI): implements(IVerifierURI) @@ -487,6 +521,10 @@ def get_filenode_cap(self): return self._filenode_uri + def is_mutable(self): + return False + + class ImmutableDirectoryURIVerifier(DirectoryURIVerifier): implements(IVerifierURI) BASE_STRING='URI:DIR2-CHK-Verifier:' @@ -494,68 +532,133 @@ BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-CHK-VERIFIER'+SEP) INNER_URI_CLASS=CHKFileVerifierURI + class UnknownURI: - def __init__(self, uri): + def __init__(self, uri, error=None): self._uri = uri + self._error = error + def to_string(self): return self._uri -def from_string(s): - if not isinstance(s, str): - raise TypeError("unknown URI type: %s.." % str(s)[:100]) - elif s.startswith('URI:CHK:'): + def get_readonly(self): + return None + + def get_error(self): + return self._error + + +ALLEGED_READONLY_PREFIX = 'ro.' +ALLEGED_IMMUTABLE_PREFIX = 'imm.' + +def from_string(u, deep_immutable=False, name=u""): + if not isinstance(u, str): + raise TypeError("unknown URI type: %s.." % str(u)[:100]) + + # We allow and check ALLEGED_READONLY_PREFIX or ALLEGED_IMMUTABLE_PREFIX + # on all URIs, even though we would only strictly need to do so for caps of + # new formats (post Tahoe-LAFS 1.6). URIs that are not consistent with their + # prefix are treated as unknown. This should be revisited when we add the + # new cap formats. See . + s = u + can_be_mutable = can_be_writeable = not deep_immutable + if s.startswith(ALLEGED_IMMUTABLE_PREFIX): + can_be_mutable = can_be_writeable = False + s = s[len(ALLEGED_IMMUTABLE_PREFIX):] + elif s.startswith(ALLEGED_READONLY_PREFIX): + can_be_writeable = False + s = s[len(ALLEGED_READONLY_PREFIX):] + + error = None + if s.startswith('URI:CHK:'): return CHKFileURI.init_from_string(s) elif s.startswith('URI:CHK-Verifier:'): return CHKFileVerifierURI.init_from_string(s) elif s.startswith('URI:LIT:'): return LiteralFileURI.init_from_string(s) elif s.startswith('URI:SSK:'): - return WriteableSSKFileURI.init_from_string(s) + if can_be_writeable: + return WriteableSSKFileURI.init_from_string(s) + error = MustBeReadonlyError("URI:SSK file writecap used in a read-only context", + name) elif s.startswith('URI:SSK-RO:'): - return ReadonlySSKFileURI.init_from_string(s) + if can_be_mutable: + return ReadonlySSKFileURI.init_from_string(s) + error = MustBeDeepImmutableError("URI:SSK-RO readcap to a mutable file used in an immutable context", + name) elif s.startswith('URI:SSK-Verifier:'): return SSKVerifierURI.init_from_string(s) elif s.startswith('URI:DIR2:'): - return DirectoryURI.init_from_string(s) + if can_be_writeable: + return DirectoryURI.init_from_string(s) + error = MustBeReadonlyError("URI:DIR2 directory writecap used in a read-only context", + name) elif s.startswith('URI:DIR2-RO:'): - return ReadonlyDirectoryURI.init_from_string(s) + if can_be_mutable: + return ReadonlyDirectoryURI.init_from_string(s) + error = MustBeDeepImmutableError("URI:DIR2-RO readcap to a mutable directory used in an immutable context", + name) elif s.startswith('URI:DIR2-Verifier:'): return DirectoryURIVerifier.init_from_string(s) elif s.startswith('URI:DIR2-CHK:'): return ImmutableDirectoryURI.init_from_string(s) elif s.startswith('URI:DIR2-LIT:'): return LiteralDirectoryURI.init_from_string(s) - return UnknownURI(s) + elif s.startswith('x-tahoe-future-test-writeable:') and not can_be_writeable: + # For testing how future writeable caps would behave in read-only contexts. + error = MustBeReadonlyError("x-tahoe-future-test-writeable: testing cap used in a read-only context", + name) + elif s.startswith('x-tahoe-future-test-mutable:') and not can_be_mutable: + # For testing how future mutable readcaps would behave in immutable contexts. + error = MustBeDeepImmutableError("x-tahoe-future-test-mutable: testing cap used in an immutable context", + name) + + #if error: print error + return UnknownURI(u, error=error) def is_uri(s): try: - from_string(s) + from_string(s, deep_immutable=False) return True except (TypeError, AssertionError): return False -def from_string_dirnode(s): - u = from_string(s) +def is_literal_file_uri(s): + if not isinstance(s, str): + return False + return (s.startswith('URI:LIT:') or + s.startswith(ALLEGED_READONLY_PREFIX + 'URI:LIT:') or + s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:LIT:')) + +def has_uri_prefix(s): + if not isinstance(s, str): + return False + return (s.startswith("URI:") or + s.startswith(ALLEGED_READONLY_PREFIX + 'URI:') or + s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:')) + +def from_string_dirnode(s, **kwargs): + u = from_string(s, **kwargs) assert IDirnodeURI.providedBy(u) return u registerAdapter(from_string_dirnode, str, IDirnodeURI) -def from_string_filenode(s): - u = from_string(s) +def from_string_filenode(s, **kwargs): + u = from_string(s, **kwargs) assert IFileURI.providedBy(u) return u registerAdapter(from_string_filenode, str, IFileURI) -def from_string_mutable_filenode(s): - u = from_string(s) +def from_string_mutable_filenode(s, **kwargs): + u = from_string(s, **kwargs) assert IMutableFileURI.providedBy(u) return u registerAdapter(from_string_mutable_filenode, str, IMutableFileURI) -def from_string_verifier(s): - u = from_string(s) +def from_string_verifier(s, **kwargs): + u = from_string(s, **kwargs) assert IVerifierURI.providedBy(u) return u registerAdapter(from_string_verifier, str, IVerifierURI) diff -rN -u old-tahoe/src/allmydata/web/common.py new-tahoe/src/allmydata/web/common.py --- old-tahoe/src/allmydata/web/common.py 2010-01-24 05:52:02.392000000 +0000 +++ new-tahoe/src/allmydata/web/common.py 2010-01-24 05:52:06.593000000 +0000 @@ -8,7 +8,8 @@ from nevow.util import resource_filename from allmydata.interfaces import ExistingChildError, NoSuchChildError, \ FileTooLargeError, NotEnoughSharesError, NoSharesError, \ - NotDeepImmutableError, EmptyPathnameComponentError + EmptyPathnameComponentError, MustBeDeepImmutableError, \ + MustBeReadonlyError, MustNotBeUnknownRWError from allmydata.mutable.common import UnrecoverableFileError from allmydata.util import abbreviate # TODO: consolidate @@ -181,9 +182,42 @@ "failure, or disk corruption. You should perform a filecheck on " "this object to learn more.") return (t, http.GONE) - if f.check(NotDeepImmutableError): - t = ("NotDeepImmutableError: a mkdir-immutable operation was given " - "a child that was not itself immutable: %s" % (f.value,)) + if f.check(MustNotBeUnknownRWError): + name = f.value.args[1] + immutable = f.value.args[2] + if immutable: + t = ("MustNotBeUnknownRWError: an operation to add a child named " + "'%s' to a directory was given an unknown cap in a write slot.\n" + "If the cap is actually an immutable readcap, then using a " + "webapi server that supports a later version of Tahoe may help.\n\n" + "If you are using the webapi directly, then specifying an immutable " + "readcap in the read slot (ro_uri) of the JSON PROPDICT, and " + "omitting the write slot (rw_uri), would also work in this " + "case.") % name.encode("utf-8") + else: + t = ("MustNotBeUnknownRWError: an operation to add a child named " + "'%s' to a directory was given an unknown cap in a write slot.\n" + "Using a webapi server that supports a later version of Tahoe " + "may help.\n\n" + "If you are using the webapi directly, specifying a readcap in " + "the read slot (ro_uri) of the JSON PROPDICT, as well as a " + "writecap in the write slot if desired, would also work in this " + "case.") % name.encode("utf-8") + return (t, http.BAD_REQUEST) + if f.check(MustBeDeepImmutableError): + name = f.value.args[1] + t = ("MustBeDeepImmutableError: a cap passed to this operation for " + "the child named '%s', needed to be immutable but was not. Either " + "the cap is being added to an immutable directory, or it was " + "originally retrieved from an immutable directory as an unknown " + "cap." % name.encode("utf-8")) + return (t, http.BAD_REQUEST) + if f.check(MustBeReadonlyError): + name = f.value.args[1] + t = ("MustBeReadonlyError: a cap passed to this operation for " + "the child named '%s', needed to be read-only but was not. " + "The cap is being passed in a read slot (ro_uri), or was retrieved " + "from a read slot as an unknown cap." % name.encode("utf-8")) return (t, http.BAD_REQUEST) if f.check(WebError): return (f.value.text, f.value.code) diff -rN -u old-tahoe/src/allmydata/web/directory.py new-tahoe/src/allmydata/web/directory.py --- old-tahoe/src/allmydata/web/directory.py 2010-01-24 05:52:02.456000000 +0000 +++ new-tahoe/src/allmydata/web/directory.py 2010-01-24 05:52:06.612000000 +0000 @@ -351,7 +351,12 @@ charset = get_arg(req, "_charset", "utf-8") name = name.decode(charset) replace = boolean_of_arg(get_arg(req, "replace", "true")) - d = self.node.set_uri(name, childcap, childcap, overwrite=replace) + + # We mustn't pass childcap for the readcap argument because we don't + # know whether it is a read cap. Passing a read cap as the writecap + # argument will work (it ends up calling NodeMaker.create_from_cap, + # which derives a readcap if necessary and possible). + d = self.node.set_uri(name, childcap, None, overwrite=replace) d.addCallback(lambda res: childcap) return d @@ -362,9 +367,9 @@ # won't show up in the resulting encoded form.. the 'name' # field is completely missing. So to allow deletion of an # empty file, we have to pretend that None means ''. The only - # downide of this is a slightly confusing error message if + # downside of this is a slightly confusing error message if # someone does a POST without a name= field. For our own HTML - # thisn't a big deal, because we create the 'delete' POST + # this isn't a big deal, because we create the 'delete' POST # buttons ourselves. name = '' charset = get_arg(req, "_charset", "utf-8") @@ -584,7 +589,11 @@ def render_title(self, ctx, data): si_s = abbreviated_dirnode(self.node) header = ["Tahoe-LAFS - Directory SI=%s" % si_s] - if self.node.is_readonly(): + if self.node.is_unknown(): + header.append(" (unknown)") + elif not self.node.is_mutable(): + header.append(" (immutable)") + elif self.node.is_readonly(): header.append(" (read-only)") else: header.append(" (modifiable)") @@ -593,7 +602,11 @@ def render_header(self, ctx, data): si_s = abbreviated_dirnode(self.node) header = ["Tahoe-LAFS Directory SI=", T.span(class_="data-chars")[si_s]] - if self.node.is_readonly(): + if self.node.is_unknown(): + header.append(" (unknown)") + elif not self.node.is_mutable(): + header.append(" (immutable)") + elif self.node.is_readonly(): header.append(" (read-only)") return ctx.tag[header] @@ -602,7 +615,7 @@ return T.div[T.a(href=link)["Return to Welcome page"]] def render_show_readonly(self, ctx, data): - if self.node.is_readonly(): + if self.node.is_unknown() or self.node.is_readonly(): return "" rocap = self.node.get_readonly_uri() root = get_root(ctx) @@ -629,7 +642,7 @@ root = get_root(ctx) here = "%s/uri/%s/" % (root, urllib.quote(self.node.get_uri())) - if self.node.is_readonly(): + if self.node.is_unknown() or self.node.is_readonly(): delete = "-" rename = "-" else: @@ -677,8 +690,8 @@ ctx.fillSlots("times", times) assert IFilesystemNode.providedBy(target), target - writecap = target.get_uri() or "" - quoted_uri = urllib.quote(writecap, safe="") # escape slashes too + target_uri = target.get_uri() or "" + quoted_uri = urllib.quote(target_uri, safe="") # escape slashes too if IMutableFileNode.providedBy(target): # to prevent javascript in displayed .html files from stealing a @@ -707,7 +720,7 @@ elif IDirectoryNode.providedBy(target): # directory - uri_link = "%s/uri/%s/" % (root, urllib.quote(writecap)) + uri_link = "%s/uri/%s/" % (root, urllib.quote(target_uri)) ctx.fillSlots("filename", T.a(href=uri_link)[html.escape(name)]) if not target.is_mutable(): @@ -794,35 +807,30 @@ kids = {} for name, (childnode, metadata) in children.iteritems(): assert IFilesystemNode.providedBy(childnode), childnode - rw_uri = childnode.get_uri() + rw_uri = childnode.get_write_uri() ro_uri = childnode.get_readonly_uri() if IFileNode.providedBy(childnode): - if childnode.is_readonly(): - rw_uri = None kiddata = ("filenode", {'size': childnode.get_size(), 'mutable': childnode.is_mutable(), }) elif IDirectoryNode.providedBy(childnode): - if childnode.is_readonly(): - rw_uri = None kiddata = ("dirnode", {'mutable': childnode.is_mutable()}) else: kiddata = ("unknown", {}) + kiddata[1]["metadata"] = metadata - if ro_uri: - kiddata[1]["ro_uri"] = ro_uri if rw_uri: kiddata[1]["rw_uri"] = rw_uri + if ro_uri: + kiddata[1]["ro_uri"] = ro_uri verifycap = childnode.get_verify_cap() if verifycap: kiddata[1]['verify_uri'] = verifycap.to_string() + kids[name] = kiddata - if dirnode.is_readonly(): - drw_uri = None - dro_uri = dirnode.get_uri() - else: - drw_uri = dirnode.get_uri() - dro_uri = dirnode.get_readonly_uri() + + drw_uri = dirnode.get_write_uri() + dro_uri = dirnode.get_readonly_uri() contents = { 'children': kids } if dro_uri: contents['ro_uri'] = dro_uri @@ -833,13 +841,14 @@ contents['verify_uri'] = verifycap.to_string() contents['mutable'] = dirnode.is_mutable() data = ("dirnode", contents) - return simplejson.dumps(data, indent=1) + "\n" + json = simplejson.dumps(data, indent=1) + "\n" + #print json + return json d.addCallback(_got) d.addCallback(text_plain, ctx) return d - def DirectoryURI(ctx, dirnode): return text_plain(dirnode.get_uri(), ctx) @@ -1131,18 +1140,39 @@ self.req.write(j+"\n") return "" -class UnknownNodeHandler(RenderMixin, rend.Page): +class UnknownNodeHandler(RenderMixin, rend.Page): def __init__(self, client, node, parentnode=None, name=None): rend.Page.__init__(self) assert node self.node = node + self.parentnode = parentnode + self.name = name def render_GET(self, ctx): req = IRequest(ctx) t = get_arg(req, "t", "").strip() if t == "info": return MoreInfo(self.node) - raise WebError("GET unknown URI type: can only do t=info, not t=%s" % t) - - + if t == "json": + if self.parentnode and self.name: + d = self.parentnode.get_metadata_for(self.name) + else: + d = defer.succeed(None) + d.addCallback(lambda md: UnknownJSONMetadata(ctx, self.node, md)) + return d + raise WebError("GET unknown URI type: can only do t=info and t=json, not t=%s.\n" + "Using a webapi server that supports a later version of Tahoe " + "may help." % t) + +def UnknownJSONMetadata(ctx, filenode, edge_metadata): + rw_uri = filenode.get_write_uri() + ro_uri = filenode.get_readonly_uri() + data = ("unknown", {}) + if ro_uri: + data[1]['ro_uri'] = ro_uri + if rw_uri: + data[1]['rw_uri'] = rw_uri + if edge_metadata is not None: + data[1]['metadata'] = edge_metadata + return text_plain(simplejson.dumps(data, indent=1) + "\n", ctx) diff -rN -u old-tahoe/src/allmydata/web/filenode.py new-tahoe/src/allmydata/web/filenode.py --- old-tahoe/src/allmydata/web/filenode.py 2010-01-24 05:52:02.484000000 +0000 +++ new-tahoe/src/allmydata/web/filenode.py 2010-01-24 05:52:06.628000000 +0000 @@ -6,10 +6,9 @@ from nevow import url, rend from nevow.inevow import IRequest -from allmydata.interfaces import ExistingChildError, CannotPackUnknownNodeError +from allmydata.interfaces import ExistingChildError from allmydata.monitor import Monitor from allmydata.immutable.upload import FileHandle -from allmydata.unknown import UnknownNode from allmydata.util import log, base32 from allmydata.web.common import text_plain, WebError, RenderMixin, \ @@ -20,7 +19,6 @@ from allmydata.web.info import MoreInfo class ReplaceMeMixin: - def replace_me_with_a_child(self, req, client, replace): # a new file is being uploaded in our place. mutable = boolean_of_arg(get_arg(req, "mutable", "false")) @@ -55,14 +53,7 @@ def replace_me_with_a_childcap(self, req, client, replace): req.content.seek(0) childcap = req.content.read() - childnode = client.create_node_from_uri(childcap, childcap+"readonly") - if isinstance(childnode, UnknownNode): - # don't be willing to pack unknown nodes: we might accidentally - # put some write-authority into the rocap slot because we don't - # know how to diminish the URI they gave us. We don't even know - # if they gave us a readcap or a writecap. - msg = "cannot attach unknown node as child %s" % str(self.name) - raise CannotPackUnknownNodeError(msg) + childnode = client.create_node_from_uri(childcap, None, name=self.name) d = self.parentnode.set_node(self.name, childnode, overwrite=replace) d.addCallback(lambda res: childnode.get_uri()) return d @@ -426,12 +417,8 @@ def FileJSONMetadata(ctx, filenode, edge_metadata): - if filenode.is_readonly(): - rw_uri = None - ro_uri = filenode.get_uri() - else: - rw_uri = filenode.get_uri() - ro_uri = filenode.get_readonly_uri() + rw_uri = filenode.get_write_uri() + ro_uri = filenode.get_readonly_uri() data = ("filenode", {}) data[1]['size'] = filenode.get_size() if ro_uri: diff -rN -u old-tahoe/src/allmydata/web/info.py new-tahoe/src/allmydata/web/info.py --- old-tahoe/src/allmydata/web/info.py 2010-01-24 05:52:02.499000000 +0000 +++ new-tahoe/src/allmydata/web/info.py 2010-01-24 05:52:06.636000000 +0000 @@ -21,6 +21,8 @@ def get_type(self): node = self.original if IDirectoryNode.providedBy(node): + if not node.is_mutable(): + return "immutable directory" return "directory" if IFileNode.providedBy(node): si = node.get_storage_index() @@ -28,7 +30,7 @@ if node.is_mutable(): return "mutable file" return "immutable file" - return "LIT file" + return "immutable LIT file" return "unknown" def render_title(self, ctx, data): @@ -68,10 +70,10 @@ def render_directory_writecap(self, ctx, data): node = self.original - if node.is_readonly(): - return "" if not IDirectoryNode.providedBy(node): return "" + if node.is_readonly(): + return "" return ctx.tag[node.get_uri()] def render_directory_readcap(self, ctx, data): @@ -86,27 +88,24 @@ return "" return ctx.tag[node.get_verify_cap().to_string()] - def render_file_writecap(self, ctx, data): node = self.original if IDirectoryNode.providedBy(node): node = node._node - if ((IDirectoryNode.providedBy(node) or IFileNode.providedBy(node)) - and node.is_readonly()): - return "" - writecap = node.get_uri() - if not writecap: + write_uri = node.get_write_uri() + #print "write_uri = %r, node = %r" % (write_uri, node) + if not write_uri: return "" - return ctx.tag[writecap] + return ctx.tag[write_uri] def render_file_readcap(self, ctx, data): node = self.original if IDirectoryNode.providedBy(node): node = node._node - readcap = node.get_readonly_uri() - if not readcap: + read_uri = node.get_readonly_uri() + if not read_uri: return "" - return ctx.tag[readcap] + return ctx.tag[read_uri] def render_file_verifycap(self, ctx, data): node = self.original diff -rN -u old-tahoe/src/allmydata/web/root.py new-tahoe/src/allmydata/web/root.py --- old-tahoe/src/allmydata/web/root.py 2010-01-24 05:52:02.625000000 +0000 +++ new-tahoe/src/allmydata/web/root.py 2010-01-24 05:52:06.710000000 +0000 @@ -12,7 +12,7 @@ from allmydata import get_package_versions_string from allmydata import provisioning from allmydata.util import idlib, log -from allmydata.interfaces import IFileNode, UnhandledCapTypeError +from allmydata.interfaces import IFileNode from allmydata.web import filenode, directory, unlinked, status, operations from allmydata.web import reliability, storage from allmydata.web.common import abbreviate_size, getxmlfile, WebError, \ @@ -85,7 +85,7 @@ try: node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) - except (TypeError, UnhandledCapTypeError, AssertionError): + except (TypeError, AssertionError): raise WebError("'%s' is not a valid file- or directory- cap" % name) @@ -104,7 +104,7 @@ # 'name' must be a file URI try: node = self.client.create_node_from_uri(name) - except (TypeError, UnhandledCapTypeError, AssertionError): + except (TypeError, AssertionError): # I think this can no longer be reached raise WebError("'%s' is not a valid file- or directory- cap" % name)