move logic for install-to-prefixdir and test-from-prefixdir from buildmaster config to misc/build_helpers

[zooko]

Buildmaster config is a bad place to keep logic—almost nobody can see it except for a privileged few, and it tends to be ill-documented and ill-organized. Move the install-to-prefix step (below) and the test-from-prefix step (below) from buildmaster config into a script in the tahoe-lafs source tree. Also add a test to it that goes red if the install puts anything other than allmydata into the Python lib dir (currently the install puts buildtest into there, which is not intended and will be fixed as soon as we have a test of it).

install-to-prefix:

class InstallToPrefixDir(PythonCommand):
    """
    Step to install into a temporary install directory using --prefix.
    """

    flunkOnFailure = False
    description = ["install-to-prefix"]
    name = "install-to-prefix"

    def __init__(self, prefixinstalldir="prefixinstalldir", *args, **kwargs):
        python_command = ["-c", ("import subprocess, sys;"
                             "sys.exit(subprocess.call([sys.executable, 'setup.py', 'install', '--single-version-externally-managed', '--record=record.txt', '--prefix', '"+prefixinstalldir+"']))")]
        kwargs['python_command'] = python_command
        PythonCommand.__init__(self, *args, **kwargs)
        self.addFactoryArguments(prefixinstalldir=prefixinstalldir)

example: ​http://tahoe-lafs.org/buildbot/builders/Kyle%20OpenBSD%20amd64/builds/24/steps/install-to-prefix/logs/stdio

test-from-prefix:

class TestFromPrefixDir(PythonCommand):
    """
    Step to run the Tahoe-LAFS tests from a --prefix=installdir installation.
    """
    flunkOnFailure = True
    description = ["test-from-prefixdir"]
    name = "test-from-prefixdir"

    def __init__(self, testsuite=None, prefixinstalldir="prefixinstalldir", srcbasedir=".", *args, **kwargs):
        if testsuite:
            assert isinstance(testsuite, basestring)
            pcmd = (
                    "import copy,os,subprocess,sys;"
                    "trial=os.path.join(os.getcwd(), 'misc', 'build_helpers', 'run_trial.py');"
                    "os.chdir('"+srcbasedir+"');"
                    "testsuite='"+testsuite+"';"
                    "prefixinstdir=os.path.join(os.getcwd(), 'prefixinstalldir');"
                    "libdir=(('win32' in sys.platform.lower()) and os.path.join(os.getcwd(), 'prefixinstalldir', 'Lib', 'site-packages') or os.path.join(os.getcwd(), 'prefixinstalldir', 'lib', 'python%(ver)s' % {'ver': sys.version[:3]}, 'site-packages'));"
                    "bindir=('win32' in sys.platform.lower()) and 'Scripts' or 'bin';"
                    "env=copy.copy(os.environ);"
                    "env['PATH']=bindir+os.pathsep+env.get('PATH','');"
                    "env['PYTHONPATH']=libdir+os.pathsep+env.get('PYTHONPATH','');"
                    "os.chdir('prefixinstalldir');"
                    "sys.exit(subprocess.call([sys.executable, trial, '--reporter=bwverbose', '--rterror', testsuite], env=env))")
        else:
            pcmd = (
                    "import copy,os,subprocess,sys;"
                    "trial=os.path.join(os.getcwd(), 'misc', 'build_helpers', 'run_trial.py');"
                    "os.chdir('"+srcbasedir+"');"
                    "testsuite=subprocess.Popen([sys.executable, 'setup.py', '--name'], stdout=subprocess.PIPE).communicate()[0].strip()+'.test';"
                    "prefixinstdir=os.path.join(os.getcwd(), 'prefixinstalldir');"
                    "libdir=(('win32' in sys.platform.lower()) and os.path.join(os.getcwd(), 'prefixinstalldir', 'Lib', 'site-packages') or os.path.join(os.getcwd(), 'prefixinstalldir', 'lib', 'python%(ver)s' % {'ver': sys.version[:3]}, 'site-packages'));"
                    "bindir=('win32' in sys.platform.lower()) and 'Scripts' or 'bin';"
                    "env=copy.copy(os.environ);"
                    "env['PATH']=bindir+os.pathsep+env.get('PATH','');"
                    "env['PYTHONPATH']=libdir+os.pathsep+env.get('PYTHONPATH','');"
                    "os.chdir('prefixinstalldir');"
                    "sys.exit(subprocess.call([sys.executable, trial, '--reporter=bwverbose', '--rterror', testsuite], env=env))")
        python_command = ["-c", pcmd]
        logfiles = {"test.log": prefixinstalldir+"/_trial_temp/test.log"}
        kwargs['python_command'] = python_command
        kwargs['logfiles'] = logfiles
        PythonCommand.__init__(self, *args, **kwargs)
        self.addFactoryArguments(testsuite=testsuite, prefixinstalldir=prefixinstalldir, srcbasedir=srcbasedir)

example: ​http://tahoe-lafs.org/buildbot/builders/Kyle%20OpenBSD%20amd64/builds/24/steps/test-from-prefixdir/logs/stdio

(Note: we also need to copy the in-source-tree buildstep script into zfec and pycryptopp source trees.)

[davidsarah]

Duplicate of #1248 (I'll copy the description to that ticket).