import/export automated testing using CMake's CTest

Will need to write full docs on this on the wiki.
basic info.
- 21 tests, OBJ/3DS/X3D/FBX, 3 tests per format import export. STL, PLY, BVH are TODO.
- uses files in ../lib/tests (checkout separate)
- run with CMake Makefiles "make test" or "ctest"
- currently checks against basic MD5 hash on scene import and file MD5 hash on export (realize this wont work predictably on binary formats *TODO*).
- currently uses a generic script for all tests with arguments to specify command to run, expected output, testing method, files to check against etc.

Has already proved useful, found a number of bugs in import export and some in blender too.
This commit is contained in:
Campbell Barton 2011-01-21 00:06:30 +00:00
parent df6bb34c2b
commit 8a3beb0012
4 changed files with 401 additions and 0 deletions

View File

@ -50,6 +50,8 @@ cmake_minimum_required(VERSION 2.6)
project(Blender)
enable_testing()
#-----------------------------------------------------------------------------
# Redirect output files

View File

@ -34,3 +34,5 @@ endif()
if(WINDOWS)
add_subdirectory(icons)
endif()
add_subdirectory(test)

202
source/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,202 @@
# -*- mode: cmake; indent-tabs-mode: t; -*-
# $Id: CMakeLists.txt 34198 2011-01-09 15:12:08Z campbellbarton $
# ***** BEGIN GPL LICENSE BLOCK *****
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Contributor(s): Jacques Beaurain.
#
# ***** END GPL LICENSE BLOCK *****
# --env-system-scripts allows to run without WITH_INSTALL
# Use '--write-blend=/tmp/test.blend' to view output
set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../lib/tests)
set(TEST_OUT_DIR ${CMAKE_BINARY_DIR}/tests)
#~ if(NOT IS_DIRECTORY ${TEST_SRC_DIR})
#~ message(FATAL_ERROR "CMake test directory not found!")
#~ endif()
# all calls to blender use this
set(GENERIC_ARGS --background --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
# OBJ Import tests
add_test(import_obj_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/cube.obj'\)
--md5=4d090508b812b5e08168aa2614746bda --md5_method=SCENE
)
add_test(import_obj_nurbs_cyclic ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/nurbs_cyclic.obj'\)
--md5=9e0da7b65b4c4f818a203d56af2d3a4b --md5_method=SCENE
--write-blend=/root/foo99.blend
)
add_test(import_obj_makehuman ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.obj\(filepath='${TEST_SRC_DIR}/io_tests/obj/makehuman.obj'\)
--md5=e0829dc078b0789e1d81f1071235bc4f --md5_method=SCENE
)
# OBJ Export tests
add_test(export_obj_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_cube.obj',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_obj_cube.obj
--md5_source=${TEST_OUT_DIR}/export_obj_cube.mtl
--md5=70bdc394c2726203ad26c085176e3484 --md5_method=FILE
)
add_test(export_obj_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_nurbs.obj',use_selection=False,use_nurbs=True\)
--md5_source=${TEST_OUT_DIR}/export_obj_nurbs.obj
--md5_source=${TEST_OUT_DIR}/export_obj_nurbs.mtl
--md5=a733ae4fa4a591ea9b0912da3af042de --md5_method=FILE
)
add_test(export_obj_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.obj\(filepath='${TEST_OUT_DIR}/export_obj_all_objects.obj',use_selection=False,use_nurbs=True\)
--md5_source=${TEST_OUT_DIR}/export_obj_all_objects.obj
--md5_source=${TEST_OUT_DIR}/export_obj_all_objects.mtl
--md5=c835899ca8993495af8a13c2f229629b --md5_method=FILE
)
# X3D Import
add_test(import_x3d_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/color_cube.x3d'\)
--md5=c80538e272812c9d765d43df269d8a9b --md5_method=SCENE
)
add_test(import_x3d_teapot ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/teapot.x3d'\)
--md5=fa19713ff71d4b3893dcbe0ab3a73955 --md5_method=SCENE
--write-blend=/root/foo99.blend
)
add_test(import_x3d_suzanne_material ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.x3d\(filepath='${TEST_SRC_DIR}/io_tests/x3d/suzanne_material.x3d'\)
--md5=52a59dcf731904ac49953dd82c020ae5 --md5_method=SCENE
)
# X3D Export
add_test(export_x3d_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_cube.x3d',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_x3d_cube.x3d
--md5=560ba3762a6604669994f661235ef93c --md5_method=FILE
)
add_test(export_x3d_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_nurbs.x3d',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_x3d_nurbs.x3d
--md5=078c0ca5a08f123cd2cdac48afb54853 --md5_method=FILE
)
add_test(export_x3d_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.x3d\(filepath='${TEST_OUT_DIR}/export_x3d_all_objects.x3d',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_x3d_all_objects.x3d
--md5=b4bddb55efd8e34af673ffb42bf4c372 --md5_method=FILE
)
# 3DS Import
add_test(import_3ds_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/cube.3ds'\)
--md5=cb5a45c35a343c3f5beca2a918472951 --md5_method=SCENE
)
add_test(import_3ds_hierarchy_lara ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/hierarchy_lara.3ds'\)
--md5=2e9812099b26ad607fdcf4c7be918c71 --md5_method=SCENE
--write-blend=/root/foo99.blend
)
add_test(import_3ds_hierarchy_greek_trireme ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.import_scene.autodesk_3ds\(filepath='${TEST_SRC_DIR}/io_tests/3ds/hierarchy_greek_trireme.3ds'\)
--md5=d05b922d7be20356d8409d1f768a3a9a --md5_method=SCENE
)
# 3DS Export
add_test(export_3ds_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_cube.3ds',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_3ds_cube.3ds
--md5=0df6cfb130052d01e31ef77d391d4cc0 --md5_method=FILE
)
add_test(export_3ds_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_nurbs.3ds',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_3ds_nurbs.3ds
--md5=ba1a6d43346fee3bcadc7e30e3c95935 --md5_method=FILE
)
add_test(export_3ds_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.autodesk_3ds\(filepath='${TEST_OUT_DIR}/export_3ds_all_objects.3ds',use_selection=False\)
--md5_source=${TEST_OUT_DIR}/export_3ds_all_objects.3ds
--md5=1523ca2e31cf7d781c7de1e17bd14520 --md5_method=FILE
)
# FBX Export
# 'use_metadata=False' for reliable md5's
add_test(export_fbx_cube ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/all_quads.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_cube.fbx',use_selection=False,use_metadata=False\)
--md5_source=${TEST_OUT_DIR}/export_fbx_cube.fbx
--md5=ce937e605e493958464d62e6de4a2f9f --md5_method=FILE
)
add_test(export_fbx_nurbs ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_geometry/nurbs.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_nurbs.fbx',use_selection=False,use_metadata=False\)
--md5_source=${TEST_OUT_DIR}/export_fbx_nurbs.fbx
--md5=e02f0147afba2a4ce1ae110567ac3531 --md5_method=FILE
)
add_test(export_fbx_all_objects ${EXECUTABLE_OUTPUT_PATH}/blender ${GENERIC_ARGS}
${TEST_SRC_DIR}/io_tests/blend_scene/all_objects.blend
--python ${CMAKE_CURRENT_LIST_DIR}/bl_test.py --
--run={'FINISHED'}&bpy.ops.export_scene.fbx\(filepath='${TEST_OUT_DIR}/export_fbx_all_objects.fbx',use_selection=False,use_metadata=False\)
--md5_source=${TEST_OUT_DIR}/export_fbx_all_objects.fbx
--md5=c29a3aa600d2e432e4a521cc1e513ba8 --md5_method=FILE
)

195
source/test/bl_test.py Normal file
View File

@ -0,0 +1,195 @@
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# <pep8 compliant>
import sys
import os
# may split this out into a new file
def replace_bpy_app_version():
""" So MD5's are pradictable from output which uses blenders versions.
"""
import bpy
app = bpy.app
app_fake = type(bpy)("bpy.app")
for attr in dir(app):
if not attr.startswith("_"):
setattr(app_fake, attr, getattr(app, attr))
app_fake.version = 0, 0, 0
app_fake.version_string = "0.00 (sub 0)"
bpy.app = app_fake
def clear_startup_blend():
import bpy
for scene in bpy.data.scenes:
for obj in scene.objects:
scene.objects.unlink(obj)
def blend_to_md5():
import bpy
scene = bpy.context.scene
ROUND = 4
def matrix2str(matrix):
return "".join([str(round(axis, ROUND)) for vector in matrix for axis in vector]).encode('ASCII')
def coords2str(seq, attr):
return "".join([str(round(axis, ROUND)) for vertex in seq for axis in getattr(vertex, attr)]).encode('ASCII')
import hashlib
md5 = hashlib.new("md5")
md5_update = md5.update
for obj in scene.objects:
md5_update(matrix2str(obj.matrix_world))
data = obj.data
if type(data) == bpy.types.Mesh:
md5_update(coords2str(data.vertices, "co"))
elif type(data) == bpy.types.Curve:
for spline in data.splines:
md5_update(coords2str(spline.bezier_points, "co"))
md5_update(coords2str(spline.points, "co"))
return md5.hexdigest()
def main():
argv = sys.argv
print(" args:", " ".join(argv))
argv = argv[argv.index("--") + 1:]
def arg_extract(arg, optional=True, array=False):
arg += "="
if array:
value = []
else:
value = None
i = 0
while i < len(argv):
if argv[i].startswith(arg):
item = argv[i][len(arg):]
del argv[i]
i -= 1
if array:
value.append(item)
else:
value = item
break
i += 1
if (not value) and (not optional):
print(" '%s' not set" % arg)
sys.exit(1)
return value
run = arg_extract("--run", optional=False)
md5 = arg_extract("--md5", optional=False)
md5_method = arg_extract("--md5_method", optional=False) # 'SCENE' / 'FILE'
# only when md5_method is 'FILE'
md5_source = arg_extract("--md5_source", optional=True, array=True)
# save blend file, for testing
write_blend = arg_extract("--write-blend", optional=True)
# ensure files are written anew
for f in md5_source:
if os.path.exists(f):
os.remove(f)
import bpy
replace_bpy_app_version()
if not bpy.data.filepath:
clear_startup_blend()
print(" Running: '%s'" % run)
print(" MD5: '%s'!" % md5)
try:
result = eval(run)
except:
import traceback
traceback.print_exc()
sys.exit(1)
if write_blend is not None:
print(" Writing Blend: %s" % write_blend)
bpy.ops.wm.save_mainfile(filepath=write_blend, check_existing=False)
print(" Result: '%s'" % str(result))
if not result:
print(" Running: %s -> False" % run)
sys.exit(1)
if md5_method == 'SCENE':
md5_new = blend_to_md5()
elif md5_method == 'FILE':
if not md5_source:
print(" Missing --md5_source argument")
sys.exit(1)
for f in md5_source:
if not os.path.exists(f):
print(" Missing --md5_source=%r argument does not point to a file")
sys.exit(1)
import hashlib
md5_instance = hashlib.new("md5")
md5_update = md5_instance.update
for f in md5_source:
md5_update(open(f, "rb").read())
md5_new = md5_instance.hexdigest()
else:
print(" Invalid --md5_method=%s argument is not a valid source")
sys.exit(1)
if md5 != md5_new:
print(" Running: %s\n MD5 Recieved: %s\n MD5 Expected: %s" % (run, md5_new, md5))
sys.exit(1)
print(" Success: %s" % run)
if __name__ == "__main__":
# So a python error exits(1)
try:
main()
except:
import traceback
traceback.print_exc()
sys.exit(1)