Buildbot updates: working windows scons build, and run slave_*.py scripts
to compile/test/package directly from svn, easier to make changes this way.
This commit is contained in:
Brecht Van Lommel
parent
5528781f79
commit
04a63cb393
|
|
@ -36,7 +36,7 @@ c['change_source'] = SVNPoller(
|
|||
# only take place on one slave.
|
||||
|
||||
from buildbot.process.factory import BuildFactory
|
||||
from buildbot.steps.source import SVN
|
||||
from buildbot.steps.source import SVN
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
from buildbot.steps.shell import Compile
|
||||
from buildbot.steps.shell import Test
|
||||
|
|
@ -50,102 +50,76 @@ from buildbot.config import BuilderConfig
|
|||
c['builders'] = []
|
||||
buildernames = []
|
||||
|
||||
|
||||
def add_builder(c, name, factory):
|
||||
def add_builder(c, name, libdir, factory):
|
||||
slavenames = []
|
||||
|
||||
for slave in master_private.slaves:
|
||||
if name in slave['builders']:
|
||||
slavenames.append(slave['name'])
|
||||
|
||||
f = factory(name)
|
||||
c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender'))
|
||||
buildernames.append(name)
|
||||
if len(slavenames) > 0:
|
||||
f = factory(name, libdir)
|
||||
c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender'))
|
||||
buildernames.append(name)
|
||||
|
||||
# common steps
|
||||
|
||||
|
||||
def svn_step():
|
||||
return SVN(baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/blender', mode='update', defaultBranch='trunk', workdir='blender')
|
||||
|
||||
|
||||
def lib_svn_step(dir):
|
||||
return SVN(name='lib svn', baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir, mode='update', defaultBranch='trunk', workdir='lib/' + dir)
|
||||
|
||||
|
||||
def cmake_compile_step():
|
||||
return Compile(command=['make'], workdir='blender')
|
||||
|
||||
|
||||
def cmake_test_step():
|
||||
return Test(workdir='blender') # make test
|
||||
|
||||
|
||||
def scons_compile_step():
|
||||
return Compile(command=['python', 'scons/scons.py'], workdir='blender')
|
||||
|
||||
|
||||
class SlavePack(ShellCommand):
|
||||
pack_script = 'slave_pack.py'
|
||||
|
||||
def start(self):
|
||||
if self.getProperty('buildername').find('scons') >= 0:
|
||||
self.setCommand(['python', pack_script, 'scons'])
|
||||
else:
|
||||
self.setCommand(['python', pack_script, 'cmake'])
|
||||
ShellCommand.start(self)
|
||||
|
||||
|
||||
def file_upload(f, id):
|
||||
def scons_file_upload(f, id):
|
||||
filename = 'buildbot_upload_' + id + '.zip'
|
||||
pack_script = 'slave_pack.py'
|
||||
unpack_script = 'master_unpack.py'
|
||||
|
||||
f.addStep(FileDownload(name='download', mastersrc=pack_script, slavedest=pack_script))
|
||||
f.addStep(ShellCommand(name='package', command=['python', pack_script], description='packaging', descriptionDone='packaged'))
|
||||
f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100 * 1024 * 1024))
|
||||
f.addStep(Compile(name='package', command=['python', 'scons/scons.py', 'BF_QUICK=slnt', 'buildslave'], description='packaging', descriptionDone='packaged', workdir='blender'))
|
||||
f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024, workdir='install'))
|
||||
f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked'))
|
||||
|
||||
# linux cmake
|
||||
# generic
|
||||
|
||||
def generic_builder(id, libdir=''):
|
||||
filename = 'buildbot_upload_' + id + '.zip'
|
||||
compile_script = '../blender/build_files/buildbot/slave_compile.py'
|
||||
test_script = '../blender/build_files/buildbot/slave_test.py'
|
||||
pack_script = '../blender/build_files/buildbot/slave_pack.py'
|
||||
unpack_script = 'master_unpack.py'
|
||||
|
||||
def linux_cmake(id):
|
||||
f = BuildFactory()
|
||||
f.addStep(svn_step())
|
||||
f.addStep(cmake_compile_step())
|
||||
f.addStep(cmake_test_step())
|
||||
file_upload(f, id)
|
||||
if libdir != '':
|
||||
f.addStep(lib_svn_step(libdir))
|
||||
|
||||
f.addStep(Compile(command=['python', compile_script, id]))
|
||||
f.addStep(Test(command=['python', test_script, id]))
|
||||
f.addStep(ShellCommand(name='package', command=['python', pack_script, id], description='packaging', descriptionDone='packaged'))
|
||||
f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024))
|
||||
f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked'))
|
||||
return f
|
||||
|
||||
add_builder(c, 'linux_x86_64_cmake', linux_cmake)
|
||||
|
||||
# mac cmake
|
||||
|
||||
|
||||
def mac_cmake(id):
|
||||
f = BuildFactory()
|
||||
f.addStep(svn_step())
|
||||
f.addStep(lib_svn_step('darwin-9.x.universal'))
|
||||
f.addStep(cmake_compile_step())
|
||||
f.addStep(cmake_test_step())
|
||||
file_upload(f, id)
|
||||
return f
|
||||
|
||||
add_builder(c, 'mac_x86_64_cmake', mac_cmake)
|
||||
add_builder(c, 'mac_x86_64_cmake', 'darwin-9.x.universal', generic_builder)
|
||||
add_builder(c, 'mac_i386_cmake', 'darwin-9.x.universal', generic_builder)
|
||||
add_builder(c, 'mac_ppc_cmake', 'darwin-9.x.universal', generic_builder)
|
||||
add_builder(c, 'linux_x86_64_cmake', '', generic_builder)
|
||||
add_builder(c, 'linux_x86_64_scons', '', generic_builder)
|
||||
|
||||
# win32 scons
|
||||
|
||||
|
||||
# TODO: add scons test target
|
||||
def win32_scons(id):
|
||||
def win32_scons(id, libdir):
|
||||
f = BuildFactory()
|
||||
f.addStep(svn_step())
|
||||
f.addStep(lib_svn_step('windows'))
|
||||
f.addStep(lib_svn_step(libdir))
|
||||
f.addStep(scons_compile_step())
|
||||
file_upload(f, id)
|
||||
scons_file_upload(f, id)
|
||||
return f
|
||||
|
||||
add_builder(c, 'win32_scons', win32_scons)
|
||||
add_builder(c, 'win32_scons', 'windows', win32_scons)
|
||||
|
||||
# SCHEDULERS
|
||||
#
|
||||
|
|
@ -178,16 +152,16 @@ c['status'] = []
|
|||
from buildbot.status import html
|
||||
from buildbot.status.web import auth, authz
|
||||
|
||||
authz_cfg = authz.Authz(
|
||||
authz_cfg=authz.Authz(
|
||||
# change any of these to True to enable; see the manual for more
|
||||
# options
|
||||
gracefulShutdown=False,
|
||||
forceBuild=True, # use this to test your slave once it is set up
|
||||
forceAllBuilds=False,
|
||||
pingBuilder=False,
|
||||
stopBuild=False,
|
||||
stopAllBuilds=False,
|
||||
cancelPendingBuild=False,
|
||||
gracefulShutdown = False,
|
||||
forceBuild = True, # use this to test your slave once it is set up
|
||||
forceAllBuilds = False,
|
||||
pingBuilder = False,
|
||||
stopBuild = False,
|
||||
stopAllBuilds = False,
|
||||
cancelPendingBuild = False,
|
||||
)
|
||||
|
||||
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
|
||||
|
|
@ -212,3 +186,4 @@ c['buildbotURL'] = "http://builder.blender.org/"
|
|||
# installations.
|
||||
|
||||
c['db_url'] = "sqlite:///state.sqlite"
|
||||
|
||||
|
|
|
|||
|
|
@ -18,69 +18,67 @@
|
|||
|
||||
# Runs on Buildbot master, to unpack incoming unload.zip into latest
|
||||
# builds directory and remove older builds.
|
||||
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
|
||||
# extension stripping
|
||||
def strip_extension(filename):
|
||||
extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe']
|
||||
extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe']
|
||||
|
||||
for ext in extensions:
|
||||
if filename.endswith(ext):
|
||||
filename = filename[:-len(ext)]
|
||||
|
||||
return filename
|
||||
for ext in extensions:
|
||||
if filename.endswith(ext):
|
||||
filename = filename[:-len(ext)]
|
||||
|
||||
return filename
|
||||
|
||||
# extract platform from package name
|
||||
def get_platform(filename):
|
||||
# name is blender-version-platform.extension. we want to get the
|
||||
# platform out, but there may be some variations, so we fiddle a
|
||||
# bit to handle current and hopefully future names
|
||||
filename = strip_extension(filename)
|
||||
filename = strip_extension(filename)
|
||||
# name is blender-version-platform.extension. we want to get the
|
||||
# platform out, but there may be some variations, so we fiddle a
|
||||
# bit to handle current and hopefully future names
|
||||
filename = strip_extension(filename)
|
||||
filename = strip_extension(filename)
|
||||
|
||||
tokens = filename.split("-")
|
||||
platforms = ['osx', 'mac', 'bsd', 'windows', 'linux', 'source', 'irix', 'solaris']
|
||||
platform_tokens = []
|
||||
found = False
|
||||
tokens = filename.split("-")
|
||||
platforms = ['osx', 'mac', 'bsd', 'win', 'linux', 'source', 'irix', 'solaris']
|
||||
platform_tokens = []
|
||||
found = False
|
||||
|
||||
for i, token in enumerate(tokens):
|
||||
if not found:
|
||||
for platform in platforms:
|
||||
if token.lower().find(platform) != -1:
|
||||
found = True
|
||||
for i, token in enumerate(tokens):
|
||||
if not found:
|
||||
for platform in platforms:
|
||||
if token.lower().find(platform) != -1:
|
||||
found = True
|
||||
|
||||
if found:
|
||||
platform_tokens += [token]
|
||||
if found:
|
||||
platform_tokens += [token]
|
||||
|
||||
return '-'.join(platform_tokens)
|
||||
return '-'.join(platform_tokens)
|
||||
|
||||
# get filename
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting file to unpack\n")
|
||||
sys.exit(1)
|
||||
sys.stderr.write("Not enough arguments, expecting file to unpack\n")
|
||||
sys.exit(1)
|
||||
|
||||
filename = sys.argv[1]
|
||||
|
||||
# open zip file
|
||||
if not os.path.exists(filename):
|
||||
sys.stderr.write("File " + filename + " not found.\n")
|
||||
sys.exit(1)
|
||||
sys.stderr.write("File " + filename + " not found.\n")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
z = zipfile.ZipFile(filename, "r")
|
||||
z = zipfile.ZipFile(filename, "r")
|
||||
except Exception, ex:
|
||||
sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
if len(z.namelist()) != 1:
|
||||
sys.stderr.write("Expected on file in " + filename + ".")
|
||||
sys.exit(1)
|
||||
sys.stderr.write("Expected on file in " + filename + ".")
|
||||
sys.exit(1)
|
||||
|
||||
package = z.namelist()[0]
|
||||
packagename = os.path.basename(package)
|
||||
|
|
@ -89,30 +87,31 @@ packagename = os.path.basename(package)
|
|||
platform = get_platform(packagename)
|
||||
|
||||
if platform == '':
|
||||
sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n')
|
||||
sys.exit(1)
|
||||
sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
# extract
|
||||
dir = 'public_html/latest_builds'
|
||||
dir = 'public_html/download'
|
||||
|
||||
try:
|
||||
zf = z.open(package)
|
||||
f = file(os.path.join(dir, packagename), "wb")
|
||||
zf = z.open(package)
|
||||
f = file(os.path.join(dir, packagename), "wb")
|
||||
|
||||
shutil.copyfileobj(zf, f)
|
||||
shutil.copyfileobj(zf, f)
|
||||
|
||||
zf.close()
|
||||
z.close()
|
||||
zf.close()
|
||||
z.close()
|
||||
except Exception, ex:
|
||||
sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
# remove other files from the same platform
|
||||
try:
|
||||
for f in os.listdir(dir):
|
||||
if f.lower().find(platform.lower()) != -1:
|
||||
if f != packagename:
|
||||
os.remove(os.path.join(dir, f))
|
||||
for f in os.listdir(dir):
|
||||
if f.lower().find(platform.lower()) != -1:
|
||||
if f != packagename:
|
||||
os.remove(os.path.join(dir, f))
|
||||
except Exception, ex:
|
||||
sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
|
||||
builder = sys.argv[1]
|
||||
|
||||
# we run from build/ directory
|
||||
blender_dir = '../blender'
|
||||
|
||||
if builder.find('cmake') != -1:
|
||||
# cmake
|
||||
|
||||
# set build options
|
||||
cmake_options = ['-DCMAKE_BUILD_TYPE:STRING=Release']
|
||||
|
||||
if builder == 'mac_x86_64_cmake':
|
||||
cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
|
||||
elif builder == 'mac_i386_cmake':
|
||||
cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=i386')
|
||||
elif builder == 'mac_ppc_cmake':
|
||||
cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=ppc')
|
||||
|
||||
# configure and make
|
||||
retcode = subprocess.call(['cmake', blender_dir] + cmake_options)
|
||||
if retcode != 0:
|
||||
sys.exit(retcode)
|
||||
retcode = subprocess.call(['make', '-s', '-j4', 'install'])
|
||||
sys.exit(retcode)
|
||||
else:
|
||||
# scons
|
||||
os.chdir(blender_dir)
|
||||
|
||||
retcode = subprocess.call(['python', 'scons/scons.py'])
|
||||
sys.exit(retcode)
|
||||
|
||||
|
|
@ -25,38 +25,36 @@ import subprocess
|
|||
import sys
|
||||
import zipfile
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
|
||||
# find release directory
|
||||
def find_release_directory():
|
||||
for d in os.listdir('.'):
|
||||
if os.path.isdir(d):
|
||||
rd = os.path.join(d, 'release')
|
||||
if os.path.exists(rd):
|
||||
return rd
|
||||
builder = sys.argv[1]
|
||||
|
||||
return None
|
||||
# scons does own packaging
|
||||
if builder.find('scons') != -1:
|
||||
os.chdir('../blender')
|
||||
retcode = subprocess.call(['python', 'scons/scons.py', 'BF_QUICK=slnt', 'buildslave'])
|
||||
sys.exit(retcode)
|
||||
|
||||
# clean release directory if it already exists
|
||||
dir = find_release_directory()
|
||||
dir = 'release'
|
||||
|
||||
if dir:
|
||||
if os.path.exists(dir):
|
||||
for f in os.listdir(dir):
|
||||
if os.path.isfile(os.path.join(dir, f)):
|
||||
os.remove(os.path.join(dir, f))
|
||||
|
||||
# create release package
|
||||
try:
|
||||
os.chdir('../blender')
|
||||
subprocess.call(['make', 'package_archive'])
|
||||
os.chdir('../build')
|
||||
except Exception, ex:
|
||||
sys.stderr.write('Make package release failed' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
# find release directory, must exist this time
|
||||
dir = find_release_directory()
|
||||
|
||||
if not dir:
|
||||
if not os.path.exists(dir):
|
||||
sys.stderr.write("Failed to find release directory.\n")
|
||||
sys.exit(1)
|
||||
|
||||
|
|
@ -85,3 +83,4 @@ try:
|
|||
except Exception, ex:
|
||||
sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
|
||||
builder = sys.argv[1]
|
||||
|
||||
# we run from build/ directory
|
||||
blender_dir = '../blender'
|
||||
|
||||
if builder.find('cmake') != -1:
|
||||
# cmake
|
||||
retcode = subprocess.call(['ctest', '.' '--output-on-failure'])
|
||||
sys.exit(retcode)
|
||||
else:
|
||||
# scons
|
||||
pass
|
||||
|
||||
Loading…
Reference in New Issue