diff --git a/tests/performance/api/config.py b/tests/performance/api/config.py index 68f4df8d487..283d1ff16ec 100644 --- a/tests/performance/api/config.py +++ b/tests/performance/api/config.py @@ -112,7 +112,7 @@ class TestConfig: self.base_dir = env.base_dir / name self.logs_dir = self.base_dir / 'logs' - config = self._read_config_module() + config = TestConfig._read_config_module(self.base_dir) self.tests = TestCollection(env, getattr(config, 'tests', ['*']), getattr(config, 'categories', ['*'])) @@ -154,10 +154,17 @@ class TestConfig: with open(config_file, 'w') as f: f.write(default_config) - def _read_config_module(self) -> None: + @staticmethod + def read_blender_executables(env, name) -> List: + config = TestConfig._read_config_module(env.base_dir / name) + builds = getattr(config, 'builds', {}) + return [pathlib.Path(build) for build in builds.values()] + + @staticmethod + def _read_config_module(base_dir: pathlib.Path) -> None: # Import config.py as a module. import importlib.util - spec = importlib.util.spec_from_file_location("testconfig", self.base_dir / 'config.py') + spec = importlib.util.spec_from_file_location("testconfig", base_dir / 'config.py') mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod @@ -195,14 +202,14 @@ class TestConfig: # Get entries for revisions based on existing builds. for revision_name, executable in self.builds.items(): - executable_path = pathlib.Path(executable) - if not executable_path.exists(): + executable_path = env._blender_executable_from_path(pathlib.Path(executable)) + if not executable_path: sys.stderr.write(f'Error: build {executable} not found\n') sys.exit(1) env.set_blender_executable(executable_path) git_hash, _ = env.run_in_blender(get_build_hash, {}) - env.unset_blender_executable() + env.set_default_blender_executable() mtime = executable_path.stat().st_mtime entries += self._get_entries(revision_name, git_hash, executable, mtime) diff --git a/tests/performance/api/environment.py b/tests/performance/api/environment.py index c9ddd493394..1a8f5ceab51 100644 --- a/tests/performance/api/environment.py +++ b/tests/performance/api/environment.py @@ -27,9 +27,10 @@ class TestEnvironment: self.git_executable = 'git' self.cmake_executable = 'cmake' self.cmake_options = ['-DWITH_INTERNATIONAL=OFF', '-DWITH_BUILDINFO=OFF'] - self.unset_blender_executable() self.log_file = None self.machine = None + self._init_default_blender_executable() + self.set_default_blender_executable() def get_machine(self, need_gpus: bool=True) -> None: if not self.machine or (need_gpus and not self.machine.has_gpus): @@ -46,7 +47,7 @@ class TestEnvironment: print(f'Init {self.base_dir}') self.base_dir.mkdir(parents=True, exist_ok=True) - if len(self.get_configs(names_only=True)) == 0: + if len(self.get_configs_names()) == 0: config_dir = self.base_dir / 'default' print(f'Creating default configuration in {config_dir}') TestConfig.write_default_config(self, config_dir) @@ -77,7 +78,7 @@ class TestEnvironment: print('Done') - def checkout(self) -> None: + def checkout(self, git_hash) -> None: # Checkout Blender revision if not self.blender_dir.exists(): sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n') @@ -87,8 +88,6 @@ class TestEnvironment: self.call([self.git_executable, 'reset', '--hard', 'HEAD'], self.blender_dir) self.call([self.git_executable, 'checkout', '--detach', git_hash], self.blender_dir) - self.build() - def build(self) -> None: # Build Blender revision if not self.build_dir.exists(): @@ -98,21 +97,54 @@ class TestEnvironment: jobs = str(multiprocessing.cpu_count()) self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir) self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir) + self._init_default_blender_executable() def set_blender_executable(self, executable_path: pathlib.Path) -> None: # Run all Blender commands with this executable. self.blender_executable = executable_path - def unset_blender_executable(self) -> None: + def _blender_executable_name(self) -> pathlib.Path: if platform.system() == "Windows": - self.blender_executable = self.build_dir / 'bin' / 'blender.exe' + return pathlib.Path('blender.exe') elif platform.system() == "Darwin": - self.blender_executable = self.build_dir / 'bin' / 'Blender.app' / 'Contents' / 'MacOS' / 'Blender' + return pathlib.Path('Blender.app') / 'Contents' / 'MacOS' / 'Blender' else: - self.blender_executable = self.build_dir / 'bin' / 'blender' + return pathlib.Path('blender') - if not self.blender_executable.exists(): - self.blender_executable = 'blender' + def _blender_executable_from_path(self, executable: pathlib.Path) -> pathlib.Path: + if executable.is_dir(): + # Directory + executable = executable / self._blender_executable_name() + elif not executable.is_file() and executable.name == 'blender': + # Executable path without proper path on Windows or macOS. + executable = executable.parent() / self._blender_executable_name() + + if executable.is_file(): + return executable + + return None + + def _init_default_blender_executable(self) -> None: + # Find a default executable to run commands independent of testing a specific build. + # Try own built executable. + built_executable = self._blender_executable_from_path(self.build_dir / 'bin') + if built_executable: + self.default_blender_executable = built_executable + return + + # Try find an executable in the configs. + for config_name in self.get_config_names(): + for executable in TestConfig.read_blender_executables(self, config_name): + executable = self._blender_executable_from_path(executable) + if executable: + self.default_blender_executable = executable + return + + # Fallback to a "blender" command in the hope it's available. + self.default_blender_executable = pathlib.Path("blender") + + def set_default_blender_executable(self) -> None: + self.blender_executable = self.default_blender_executable def set_log_file(self, filepath: pathlib.Path, clear=True) -> None: # Log all commands and output to this file. @@ -219,19 +251,27 @@ class TestEnvironment: filepaths.append(pathlib.Path(filename)) return filepaths + def get_config_names(self) -> List: + names = [] + + if self.base_dir.exists(): + for dirname in os.listdir(self.base_dir): + dirpath = self.base_dir / dirname / 'config.py' + if dirpath.exists(): + names.append(dirname) + + return names + def get_configs(self, name: str=None, names_only: bool=False) -> List: # Get list of configurations in the benchmarks directory. configs = [] - if self.base_dir.exists(): - for dirname in os.listdir(self.base_dir): - if not name or dirname == name: - dirpath = self.base_dir / dirname / 'config.py' - if dirpath.exists(): - if names_only: - configs.append(dirname) - else: - configs.append(TestConfig(self, dirname)) + for config_name in self.get_config_names(): + if not name or config_name == name: + if names_only: + configs.append(config_name) + else: + configs.append(TestConfig(self, config_name)) return configs diff --git a/tests/performance/benchmark b/tests/performance/benchmark index 3b43bd0aa96..4ca1d0eda4c 100755 --- a/tests/performance/benchmark +++ b/tests/performance/benchmark @@ -112,7 +112,7 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry env.set_blender_executable(pathlib.Path(entry.executable)) else: env.checkout(git_hash) - env.build(git_hash) + env.build() # Run test and update output and status. entry.status = 'running' @@ -126,7 +126,7 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry # Restore default logging and Blender executable. env.unset_log_file() - env.unset_blender_executable() + env.set_default_blender_executable() return True @@ -155,7 +155,7 @@ def cmd_list(env: api.TestEnvironment, argv: List) -> None: print('') print('CONFIGS') - configs = env.get_configs(names_only=True) + configs = env.get_config_names() for config_name in configs: print(config_name)