id	summary	reporter	owner	description	type	status	priority	milestone	component	version	resolution	keywords	cc	launchpad_bug
1590	"S3 backend: intermittent ""We encountered an internal error. Please try again."" from S3"	davidsarah	secorp	"Traceback from CLI ({{{tahoe backup}}}):
{{{
Traceback (most recent call last):
  File ""/home/zooko/playground/tahoe-lafs/trunk/support/bin/tahoe"", line 9, in <module>
    load_entry_point('allmydata-tahoe==1.9.0-r5375', 'console_scripts', 'tahoe')()
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/runner.py"", line 113, in run
    rc = runner(sys.argv[1:], install_node_control=install_node_control)
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/runner.py"", line 99, in runner
    rc = cli.dispatch[command](so)
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/cli.py"", line 569, in backup
    rc = tahoe_backup.backup(options)
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/tahoe_backup.py"", line 325, in backup
    return bu.run()
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/tahoe_backup.py"", line 118, in run
    new_backup_dircap = self.process(options.from_dir)
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/tahoe_backup.py"", line 188, in process
    childcap = self.process(childpath)
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/tahoe_backup.py"", line 194, in process
    childcap, metadata = self.upload(childpath)
  File ""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/scripts/tahoe_backup.py"", line 305, in upload
    raise HTTPError(""Error during file PUT"", resp)
allmydata.scripts.common_http.HTTPError: Error during file PUT: 500 Internal Server Error
""Traceback (most recent call last):
  File \""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/util/pipeline.py\"", line 65, in _ebDeferred
    self.checkForFinished()
  File \""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/util/pipeline.py\"", line 53, in checkForFinished
    self.errback(self.failure)
  File \""/usr/local/lib/python2.7/dist-packages/Twisted-11.0.0-py2.7-linux-x86_64.egg/twisted/internet/defer.py\"", line 388, in errback
    self._startRunCallbacks(fail)
  File \""/usr/local/lib/python2.7/dist-packages/Twisted-11.0.0-py2.7-linux-x86_64.egg/twisted/internet/defer.py\"", line 455, in _startRunCallbacks
    self._runCallbacks()
--- <exception caught here> ---
  File \""/usr/local/lib/python2.7/dist-packages/Twisted-11.0.0-py2.7-linux-x86_64.egg/twisted/internet/defer.py\"", line 542, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File \""/home/zooko/playground/tahoe-lafs/trunk/src/allmydata/immutable/encode.py\"", line 504, in _remove_shareholder
    raise UploadUnhappinessError(msg)
allmydata.interfaces.UploadUnhappinessError: shares could be placed or found on only 0 server(s).
 We were asked to place shares on at least 1 server(s) such that any 1 of them have enough shares to recover the file.:
 [Failure instance: Traceback (failure with no frames): <class 'allmydata.util.pipeline.PipelineError'>:
 <PipelineError error=([Failure instance: Traceback (failure with no frames): <class 'foolscap.tokens.RemoteException'>:
 <RemoteException around '[CopiedFailure instance: Traceback from remote host -- Traceback (most recent call last):
  File \""/usr/local/lib/python2.6/dist-packages/Twisted-11.1.0-py2.6-linux-i686.egg/twisted/internet/tcp.py\"", line 277, in connectionLost
    protocol.connectionLost(reason)
  File \""/usr/local/lib/python2.6/dist-packages/Twisted-11.1.0-py2.6-linux-i686.egg/twisted/web/client.py\"", line 191, in connectionLost
    self.factory._disconnectedDeferred.callback(None)
  File \""/usr/local/lib/python2.6/dist-packages/Twisted-11.1.0-py2.6-linux-i686.egg/twisted/internet/defer.py\"", line 362, in callback
    self._startRunCallbacks(result)
  File \""/usr/local/lib/python2.6/dist-packages/Twisted-11.1.0-py2.6-linux-i686.egg/twisted/internet/defer.py\"", line 458, in _startRunCallbacks
    self._runCallbacks()
--- <exception caught here> ---
  File \""/usr/local/lib/python2.6/dist-packages/Twisted-11.1.0-py2.6-linux-i686.egg/twisted/internet/defer.py\"", line 545, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File \""/usr/local/lib/python2.6/dist-packages/txAWS-0.2.1.post2-py2.6.egg/txaws/s3/client.py\"", line 30, in s3_error_wrapper
    error_wrapper(error, S3Error)
  File \""/usr/local/lib/python2.6/dist-packages/txAWS-0.2.1.post2-py2.6.egg/txaws/client/base.py\"", line 46, in error_wrapper
    raise fallback_error
txaws.s3.exception.S3Error: Error Message: We encountered an internal error. Please try again.
]'>
])>
]
""
}}}

Incident report: https://tahoe-lafs.org/~zooko/incident-2011-11-19--01%3a16%3a49Z-xrkocuy.flog.bz2"	defect	closed	critical	undecided	code-storage	1.9.0	fixed	s3-backend reliability availability preservation error