Tests: improve finding of Blender executables in benchmarking

* Allow specifying a folder and automatically setting the proper executable
  name depending on the operating system
* Use executables from configs for listing devices instead of a blender
  command being available

tests/performance/api/config.py

@@ -112,7 +112,7 @@ class TestConfig:
         self.base_dir = env.base_dir / name
         self.logs_dir = self.base_dir / 'logs'
 
-        config = self._read_config_module()
+        config = TestConfig._read_config_module(self.base_dir)
         self.tests = TestCollection(env,
                                     getattr(config, 'tests', ['*']),
                                     getattr(config, 'categories', ['*']))
@@ -154,10 +154,17 @@ class TestConfig:
         with open(config_file, 'w') as f:
             f.write(default_config)
 
-    def _read_config_module(self) -> None:
+    @staticmethod
+    def read_blender_executables(env, name) -> List:
+        config = TestConfig._read_config_module(env.base_dir / name)
+        builds = getattr(config, 'builds', {})
+        return [pathlib.Path(build) for build in builds.values()]
+
+    @staticmethod
+    def _read_config_module(base_dir: pathlib.Path) -> None:
         # Import config.py as a module.
         import importlib.util
-        spec = importlib.util.spec_from_file_location("testconfig", self.base_dir / 'config.py')
+        spec = importlib.util.spec_from_file_location("testconfig", base_dir / 'config.py')
         mod = importlib.util.module_from_spec(spec)
         spec.loader.exec_module(mod)
         return mod
@@ -195,14 +202,14 @@ class TestConfig:
 
         # Get entries for revisions based on existing builds.
         for revision_name, executable in self.builds.items():
-            executable_path = pathlib.Path(executable)
+            executable_path = env._blender_executable_from_path(pathlib.Path(executable))
-            if not executable_path.exists():
+            if not executable_path:
                 sys.stderr.write(f'Error: build {executable} not found\n')
                 sys.exit(1)
 
             env.set_blender_executable(executable_path)
             git_hash, _ = env.run_in_blender(get_build_hash, {})
-            env.unset_blender_executable()
+            env.set_default_blender_executable()
 
             mtime = executable_path.stat().st_mtime
             entries += self._get_entries(revision_name, git_hash, executable, mtime)

tests/performance/api/environment.py

@@ -27,9 +27,10 @@ class TestEnvironment:
         self.git_executable = 'git'
         self.cmake_executable = 'cmake'
         self.cmake_options = ['-DWITH_INTERNATIONAL=OFF', '-DWITH_BUILDINFO=OFF']
-        self.unset_blender_executable()
         self.log_file = None
         self.machine = None
+        self._init_default_blender_executable()
+        self.set_default_blender_executable()
 
     def get_machine(self, need_gpus: bool=True) -> None:
         if not self.machine or (need_gpus and not self.machine.has_gpus):
@@ -46,7 +47,7 @@ class TestEnvironment:
         print(f'Init {self.base_dir}')
         self.base_dir.mkdir(parents=True, exist_ok=True)
 
-        if len(self.get_configs(names_only=True)) == 0:
+        if len(self.get_configs_names()) == 0:
             config_dir = self.base_dir / 'default'
             print(f'Creating default configuration in {config_dir}')
             TestConfig.write_default_config(self, config_dir)
@@ -77,7 +78,7 @@ class TestEnvironment:
 
         print('Done')
 
-    def checkout(self) -> None:
+    def checkout(self, git_hash) -> None:
         # Checkout Blender revision
         if not self.blender_dir.exists():
             sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n')
@@ -87,8 +88,6 @@ class TestEnvironment:
         self.call([self.git_executable, 'reset', '--hard', 'HEAD'], self.blender_dir)
         self.call([self.git_executable, 'checkout', '--detach', git_hash], self.blender_dir)
 
-        self.build()
-
     def build(self) -> None:
         # Build Blender revision
         if not self.build_dir.exists():
@@ -98,21 +97,54 @@ class TestEnvironment:
         jobs = str(multiprocessing.cpu_count())
         self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir)
         self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir)
+        self._init_default_blender_executable()
 
     def set_blender_executable(self, executable_path: pathlib.Path) -> None:
         # Run all Blender commands with this executable.
         self.blender_executable = executable_path
 
-    def unset_blender_executable(self) -> None:
+    def _blender_executable_name(self) -> pathlib.Path:
         if platform.system() == "Windows":
-            self.blender_executable = self.build_dir / 'bin' / 'blender.exe'
+            return pathlib.Path('blender.exe')
         elif platform.system() == "Darwin":
-            self.blender_executable = self.build_dir / 'bin' / 'Blender.app' / 'Contents' / 'MacOS' / 'Blender'
+            return pathlib.Path('Blender.app') / 'Contents' / 'MacOS' / 'Blender'
         else:
-            self.blender_executable = self.build_dir / 'bin' / 'blender'
+            return pathlib.Path('blender')
 
-        if not self.blender_executable.exists():
+    def _blender_executable_from_path(self, executable: pathlib.Path) -> pathlib.Path:
-            self.blender_executable = 'blender'
+        if executable.is_dir():
+            # Directory
+            executable = executable / self._blender_executable_name()
+        elif not executable.is_file() and executable.name == 'blender':
+            # Executable path without proper path on Windows or macOS.
+            executable = executable.parent() / self._blender_executable_name()
+
+        if executable.is_file():
+            return executable
+
+        return None
+
+    def _init_default_blender_executable(self) -> None:
+        # Find a default executable to run commands independent of testing a specific build.
+        # Try own built executable.
+        built_executable = self._blender_executable_from_path(self.build_dir / 'bin')
+        if built_executable:
+            self.default_blender_executable = built_executable
+            return
+
+        # Try find an executable in the configs.
+        for config_name in self.get_config_names():
+            for executable in TestConfig.read_blender_executables(self, config_name):
+                executable = self._blender_executable_from_path(executable)
+                if executable:
+                    self.default_blender_executable = executable
+                    return
+
+        # Fallback to a "blender" command in the hope it's available.
+        self.default_blender_executable = pathlib.Path("blender")
+
+    def set_default_blender_executable(self) -> None:
+        self.blender_executable = self.default_blender_executable
 
     def set_log_file(self, filepath: pathlib.Path, clear=True) -> None:
         # Log all commands and output to this file.
@@ -219,19 +251,27 @@ class TestEnvironment:
             filepaths.append(pathlib.Path(filename))
         return filepaths
 
+    def get_config_names(self) -> List:
+        names = []
+
+        if self.base_dir.exists():
+            for dirname in os.listdir(self.base_dir):
+                dirpath = self.base_dir / dirname / 'config.py'
+                if dirpath.exists():
+                    names.append(dirname)
+
+        return names
+
     def get_configs(self, name: str=None, names_only: bool=False) -> List:
         # Get list of configurations in the benchmarks directory.
         configs = []
 
-        if self.base_dir.exists():
+        for config_name in self.get_config_names():
-            for dirname in os.listdir(self.base_dir):
+            if not name or config_name == name:
-                if not name or dirname == name:
+                if names_only:
-                    dirpath = self.base_dir / dirname / 'config.py'
+                    configs.append(config_name)
-                    if dirpath.exists():
+                else:
-                        if names_only:
+                    configs.append(TestConfig(self, config_name))
-                            configs.append(dirname)
-                        else:
-                            configs.append(TestConfig(self, dirname))
 
         return configs
 

tests/performance/benchmark

@@ -112,7 +112,7 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
         env.set_blender_executable(pathlib.Path(entry.executable))
     else:
         env.checkout(git_hash)
-        env.build(git_hash)
+        env.build()
 
     # Run test and update output and status.
     entry.status = 'running'
@@ -126,7 +126,7 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
 
     # Restore default logging and Blender executable.
     env.unset_log_file()
-    env.unset_blender_executable()
+    env.set_default_blender_executable()
 
     return True
 
@@ -155,7 +155,7 @@ def cmd_list(env: api.TestEnvironment, argv: List) -> None:
     print('')
 
     print('CONFIGS')
-    configs = env.get_configs(names_only=True)
+    configs = env.get_config_names()
     for config_name in configs:
         print(config_name)