diff --git a/tests/performance/api/__init__.py b/tests/performance/api/__init__.py new file mode 100644 index 00000000000..6f344a41841 --- /dev/null +++ b/tests/performance/api/__init__.py @@ -0,0 +1,8 @@ +# Apache License, Version 2.0 + +from .environment import TestEnvironment +from .device import TestDevice, TestMachine +from .config import TestEntry, TestQueue, TestConfig +from .test import Test, TestCollection +from .graph import TestGraph + diff --git a/tests/performance/api/config.py b/tests/performance/api/config.py new file mode 100644 index 00000000000..900cac0a0bb --- /dev/null +++ b/tests/performance/api/config.py @@ -0,0 +1,255 @@ +# Apache License, Version 2.0 + +import fnmatch +import json +import pathlib +import sys + +from dataclasses import dataclass, field +from typing import Dict, List + +from .test import TestCollection + +def get_build_hash(args: None) -> str: + import bpy + import sys + build_hash = bpy.app.build_hash.decode('utf-8') + return '' if build_hash == 'Unknown' else build_hash + +@dataclass +class TestEntry: + """Test to run, a combination of revision, test and device.""" + test: str = '' + category: str = '' + revision: str = '' + git_hash: str = '' + executable: str = '' + date: int = 0 + device_type: str = 'CPU' + device_id: str = 'CPU' + device_name: str = 'Unknown CPU' + status: str = 'queued' + output: Dict = field(default_factory=dict) + benchmark_type: str = 'comparison' + + def to_json(self) -> Dict: + json_dict = {} + for field in self.__dataclass_fields__: + json_dict[field] = getattr(self, field) + return json_dict + + def from_json(self, json_dict): + for field in self.__dataclass_fields__: + setattr(self, field, json_dict[field]) + +class TestQueue: + """Queue of tests to be run or inspected. Matches JSON file on disk.""" + + def __init__(self, filepath: pathlib.Path): + self.filepath = filepath + self.has_multiple_revisions_to_build = False + self.has_multiple_categories = False + self.entries = [] + + if self.filepath.is_file(): + with open(self.filepath, 'r') as f: + json_entries = json.load(f) + + for json_entry in json_entries: + entry = TestEntry() + entry.from_json(json_entry) + self.entries.append(entry) + + def rows(self, use_revision_columns: bool) -> List: + # Generate rows of entries for printing and running. + entries = sorted(self.entries, key=lambda entry: + (entry.revision, + entry.device_id, + entry.category, + entry.test)) + + if not use_revision_columns: + # One entry per row. + return [[entry] for entry in entries] + else: + # Multiple revisions per row. + rows = {} + + for entry in entries: + key = (entry.device_id, entry.category, entry.test) + if key in rows: + rows[key].append(entry) + else: + rows[key] = [entry] + + return [value for _, value in sorted(rows.items())] + + def find(self, revision: str, test: str, category: str, device_id: str) -> Dict: + for entry in self.entries: + if entry.revision == revision and \ + entry.test == test and \ + entry.category == category and \ + entry.device_id == device_id: + return entry + + return None + + def write(self) -> None: + json_entries = [entry.to_json() for entry in self.entries] + with open(self.filepath, 'w') as f: + json.dump(json_entries, f, indent=2) + +class TestConfig: + """Test configuration, containing a subset of revisions, tests and devices.""" + + def __init__(self, env, name: str): + # Init configuration from config.py file. + self.name = name + self.base_dir = env.base_dir / name + self.logs_dir = self.base_dir / 'logs' + + config = self._read_config_module() + self.tests = TestCollection(env, + getattr(config, 'tests', ['*']), + getattr(config, 'categories', ['*'])) + self.revisions = getattr(config, 'revisions', {}) + self.builds = getattr(config, 'builds', {}) + self.queue = TestQueue(self.base_dir / 'results.json') + self.benchmark_type = getattr(config, 'benchmark_type', 'comparison') + + self.devices = [] + self._update_devices(env, getattr(config, 'devices', ['CPU'])) + + self._update_queue(env) + + def revision_names(self) -> List: + return sorted(list(self.revisions.keys()) + list(self.builds.keys())) + + def device_name(self, device_id: str) -> str: + for device in self.devices: + if device.id == device_id: + return device.name + + return "Unknown" + + @staticmethod + def write_default_config(env, config_dir: pathlib.Path) -> None: + config_dir.mkdir(parents=True, exist_ok=True) + + default_config = """devices = ['CPU']\n""" + default_config += """tests = ['*']\n""" + default_config += """categories = ['*']\n""" + default_config += """builds = {\n""" + default_config += """ 'master': '/home/user/blender-git/build/bin/blender',""" + default_config += """ '2.93': '/home/user/blender-2.93/blender',""" + default_config += """}\n""" + default_config += """revisions = {\n""" + default_config += """}\n""" + + config_file = config_dir / 'config.py' + with open(config_file, 'w') as f: + f.write(default_config) + + def _read_config_module(self) -> None: + # Import config.py as a module. + import importlib.util + spec = importlib.util.spec_from_file_location("testconfig", self.base_dir / 'config.py') + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def _update_devices(self, env, device_filters: List) -> None: + # Find devices matching the filters. + need_gpus = device_filters != ['CPU'] + machine = env.get_machine(need_gpus) + + self.devices = [] + for device in machine.devices: + for device_filter in device_filters: + if fnmatch.fnmatch(device.id, device_filter): + self.devices.append(device) + break + + def _update_queue(self, env) -> None: + # Update queue to match configuration, adding and removing entries + # so that there is one entry for each revision, device and test + # combination. + entries = [] + + # Get entries for specified commits, tags and branches. + for revision_name, revision_commit in self.revisions.items(): + git_hash = env.resolve_git_hash(revision_commit) + date = env.git_hash_date(git_hash) + entries += self._get_entries(revision_name, git_hash, '', date) + + # Optimization to avoid rebuilds. + revisions_to_build = set() + for entry in entries: + if entry.status in ('queued', 'outdated'): + revisions_to_build.add(entry.git_hash) + self.queue.has_multiple_revisions_to_build = len(revisions_to_build) > 1 + + # Get entries for revisions based on existing builds. + for revision_name, executable in self.builds.items(): + executable_path = pathlib.Path(executable) + if not executable_path.exists(): + sys.stderr.write(f'Error: build {executable} not found\n') + sys.exit(1) + + env.set_blender_executable(executable_path) + git_hash, _ = env.run_in_blender(get_build_hash, {}) + env.unset_blender_executable() + + mtime = executable_path.stat().st_mtime + entries += self._get_entries(revision_name, git_hash, executable, mtime) + + # Detect number of categories for more compact printing. + categories = set() + for entry in entries: + categories.add(entry.category) + self.queue.has_multiple_categories = len(categories) > 1 + + # Replace actual entries. + self.queue.entries = entries + + def _get_entries(self, + revision_name: str, + git_hash: str, + executable: pathlib.Path, + date: int) -> None: + entries = [] + for test in self.tests.tests: + test_name = test.name() + test_category = test.category() + + for device in self.devices: + entry = self.queue.find(revision_name, test_name, test_category, device.id) + if entry: + # Test if revision hash or executable changed. + if entry.git_hash != git_hash or \ + entry.executable != executable or \ + entry.benchmark_type != self.benchmark_type or \ + entry.date != date: + # Update existing entry. + entry.git_hash = git_hash + entry.executable = executable + entry.benchmark_type = self.benchmark_type + entry.date = date + if entry.status in ('done', 'failed'): + entry.status = 'outdated' + else: + # Add new entry if it did not exist yet. + entry = TestEntry( + revision=revision_name, + git_hash=git_hash, + executable=executable, + date=date, + test=test_name, + category=test_category, + device_type=device.type, + device_id=device.id, + device_name=device.name, + benchmark_type=self.benchmark_type) + entries.append(entry) + + return entries diff --git a/tests/performance/api/device.py b/tests/performance/api/device.py new file mode 100644 index 00000000000..e27540da747 --- /dev/null +++ b/tests/performance/api/device.py @@ -0,0 +1,68 @@ +# Apache License, Version 2.0 + +import platform +import subprocess +from typing import List + +def get_cpu_name() -> str: + # Get full CPU name. + if platform.system() == "Windows": + return platform.processor() + elif platform.system() == "Darwin": + cmd = ['/usr/sbin/sysctl', "-n", "machdep.cpu.brand_string"] + return subprocess.check_output(cmd).strip().decode('utf-8') + else: + with open('/proc/cpuinfo') as f: + for line in f: + if line.startswith('model name'): + return line.split(':')[1].strip() + + return "Unknown CPU" + +def get_gpu_device(args: None) -> List: + # Get the list of available Cycles GPU devices. + import bpy + import sys + + prefs = bpy.context.preferences + cprefs = prefs.addons['cycles'].preferences + + result = [] + + for device_type, _, _, _ in cprefs.get_device_types(bpy.context): + cprefs.compute_device_type = device_type + devices = cprefs.get_devices_for_type(device_type) + index = 0 + for device in devices: + if device.type == device_type: + result.append({'type': device.type, 'name': device.name, 'index': index}) + index += 1 + break + + return result + +class TestDevice: + def __init__(self, device_type: str, device_id: str, name: str, operating_system: str): + self.type = device_type + self.id = device_id + self.name = name + self.operating_system = operating_system + +class TestMachine: + def __init__(self, env, need_gpus: bool): + operating_system = platform.system() + + self.devices = [TestDevice('CPU', 'CPU', get_cpu_name(), operating_system)] + self.has_gpus = need_gpus + + if need_gpus and env.blender_executable: + gpu_devices, _ = env.run_in_blender(get_gpu_device, {}) + for gpu_device in gpu_devices: + device_type = gpu_device['type'] + device_name = gpu_device['name'] + device_id = gpu_device['type'] + "_" + str(gpu_device['index']) + self.devices.append(TestDevice(device_type, device_id, device_name, operating_system)) + + def cpu_device(self) -> TestDevice: + return self.devices[0] + diff --git a/tests/performance/api/environment.py b/tests/performance/api/environment.py new file mode 100644 index 00000000000..7c4e5e761a6 --- /dev/null +++ b/tests/performance/api/environment.py @@ -0,0 +1,243 @@ +# Apache License, Version 2.0 + +import base64 +import glob +import inspect +import multiprocessing +import os +import pathlib +import platform +import pickle +import subprocess +import sys +from typing import Callable, Dict, List + +from .config import TestConfig +from .device import TestMachine + +class TestEnvironment: + def __init__(self, blender_git_dir: pathlib.Path, base_dir: pathlib.Path): + self.blender_git_dir = blender_git_dir + self.base_dir = base_dir + self.blender_dir = base_dir / 'blender' + self.build_dir = base_dir / 'build' + self.lib_dir = base_dir / 'lib' + self.benchmarks_dir = self.blender_git_dir.parent / 'lib' / 'benchmarks' + self.git_executable = 'git' + self.cmake_executable = 'cmake' + self.cmake_options = ['-DWITH_INTERNATIONAL=OFF', '-DWITH_BUILDINFO=OFF'] + self.unset_blender_executable() + self.log_file = None + self.machine = None + + def get_machine(self, need_gpus: bool=True) -> None: + if not self.machine or (need_gpus and not self.machine.has_gpus): + self.machine = TestMachine(self, need_gpus) + + return self.machine + + def init(self, build) -> None: + if not self.benchmarks_dir.exists(): + sys.stderr.write(f'Error: benchmark files directory not found at {self.benchmarks_dir}') + sys.exit(1) + + # Create benchmarks folder contents. + print(f'Init {self.base_dir}') + self.base_dir.mkdir(parents=True, exist_ok=True) + + if len(self.get_configs(names_only=True)) == 0: + config_dir = self.base_dir / 'default' + print(f'Creating default configuration in {config_dir}') + TestConfig.write_default_config(self, config_dir) + + if build: + if not self.lib_dir.exists(): + print(f'Creating symlink at {self.lib_dir}') + self.lib_dir.symlink_to(self.blender_git_dir.parent / 'lib') + else: + print(f'Exists {self.lib_dir}') + + if not self.blender_dir.exists(): + print(f'Init git worktree in {self.blender_dir}') + self.call([self.git_executable, 'worktree', 'add', '--detach', self.blender_dir, 'HEAD'], self.blender_git_dir) + else: + print(f'Exists {self.blender_dir}') + + if not self.build_dir.exists(): + print(f'Init build in {self.build_dir}') + self.build_dir.mkdir() + # No translation to avoid dealing with submodules + self.call([self.cmake_executable, self.blender_dir, '.'] + self.cmake_options, self.build_dir) + else: + print(f'Exists {self.build_dir}') + + print("Building") + self.build() + + print('Done') + + def checkout(self) -> None: + # Checkout Blender revision + if not self.blender_dir.exists(): + sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n') + sys.exit(1) + + self.call([self.git_executable, 'clean', '-f', '-d'], self.blender_dir) + self.call([self.git_executable, 'reset', '--hard', 'HEAD'], self.blender_dir) + self.call([self.git_executable, 'checkout', '--detach', git_hash], self.blender_dir) + + self.build() + + def build(self) -> None: + # Build Blender revision + if not self.build_dir.exists(): + sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n') + sys.exit(1) + + jobs = str(multiprocessing.cpu_count()) + self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir) + self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir) + + def set_blender_executable(self, executable_path: pathlib.Path) -> None: + # Run all Blender commands with this executable. + self.blender_executable = executable_path + + def unset_blender_executable(self) -> None: + if platform.system() == "Windows": + self.blender_executable = self.build_dir / 'bin' / 'blender.exe' + elif platform.system() == "Darwin": + self.blender_executable = self.build_dir / 'bin' / 'Blender.app' / 'Contents' / 'MacOS' / 'Blender' + else: + self.blender_executable = self.build_dir / 'bin' / 'blender' + + if not self.blender_executable.exists(): + self.blender_executable = 'blender' + + def set_log_file(self, filepath: pathlib.Path, clear=True) -> None: + # Log all commands and output to this file. + self.log_file = filepath + + if clear: + self.log_file.unlink(missing_ok=True) + + def unset_log_file(self) -> None: + self.log_file = None + + def call(self, args: List[str], cwd: pathlib.Path, silent=False) -> List[str]: + # Execute command with arguments in specified directory, + # and return combined stdout and stderr output. + + # Open log file for writing + f = None + if self.log_file: + if not self.log_file.exists(): + self.log_file.parent.mkdir(parents=True, exist_ok=True) + f = open(self.log_file, 'a') + f.write('\n' + ' '.join([str(arg) for arg in args]) + '\n\n') + + proc = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + # Read line by line + lines = [] + try: + while proc.poll() is None: + line = proc.stdout.readline() + if line: + line_str = line.decode('utf-8', 'ignore') + lines.append(line_str) + if f: + f.write(line_str) + except KeyboardInterrupt: + # Avoid processes that keep running when interrupting. + proc.terminate() + + if f: + f.close() + + # Print command output on error + if proc.returncode != 0 and not silent: + for line in lines: + print(line.rstrip()) + raise Exception("Error executing command") + + return lines + + def call_blender(self, args: List[str], foreground=False) -> List[str]: + # Execute Blender command with arguments. + common_args = ['--factory-startup', '--enable-autoexec', '--python-exit-code', '1'] + if foreground: + common_args += ['--no-window-focus', '--window-geometry', '0', '0', '1024', '768'] + else: + common_args += ['--background'] + + return self.call([self.blender_executable] + common_args + args, cwd=self.base_dir) + + def run_in_blender(self, + function: Callable[[Dict], Dict], + args: Dict, + blender_args: List=[], + foreground=False) -> Dict: + # Run function in a Blender instance. Arguments and return values are + # passed as a Python object that must be serializable with pickle. + + # Get information to call this function from Blender. + package_path = pathlib.Path(__file__).parent.parent + functionname = function.__name__ + modulename = inspect.getmodule(function).__name__ + + # Serialize arguments in base64, to avoid having to escape it. + args = base64.b64encode(pickle.dumps(args)) + output_prefix = 'TEST_OUTPUT: ' + + expression = (f'import sys, pickle, base64\n' + f'sys.path.append("{package_path}")\n' + f'import {modulename}\n' + f'args = pickle.loads(base64.b64decode({args}))\n' + f'result = {modulename}.{functionname}(args)\n' + f'result = base64.b64encode(pickle.dumps(result))\n' + f'print("{output_prefix}" + result.decode())\n') + + expr_args = blender_args + ['--python-expr', expression] + lines = self.call_blender(expr_args, foreground=foreground) + + # Parse output. + for line in lines: + if line.startswith(output_prefix): + output = line[len(output_prefix):].strip() + result = pickle.loads(base64.b64decode(output)) + return result, lines + + return {}, lines + + def find_blend_files(self, dirpath: pathlib.Path) -> List: + # Find .blend files in subdirectories of the given directory in the + # lib/benchmarks directory. + dirpath = self.benchmarks_dir / dirpath + filepaths = [] + for filename in glob.iglob(str(dirpath / '*.blend'), recursive=True): + filepaths.append(pathlib.Path(filename)) + return filepaths + + def get_configs(self, name: str=None, names_only: bool=False) -> List: + # Get list of configurations in the benchmarks directory. + configs = [] + + if self.base_dir.exists(): + for dirname in os.listdir(self.base_dir): + if not name or dirname == name: + dirpath = self.base_dir / dirname / 'config.py' + if dirpath.exists(): + if names_only: + configs.append(dirname) + else: + configs.append(TestConfig(self, dirname)) + + return configs + + def resolve_git_hash(self, revision): + # Get git hash for a tag or branch. + return self.call([self.git_executable, 'rev-parse', revision], self.blender_git_dir)[0].strip() + + def git_hash_date(self, git_hash): + # Get commit data for a git hash. + return int(self.call([self.git_executable, 'log', '-n1', git_hash, '--format=%at'], self.blender_git_dir)[0].strip()) diff --git a/tests/performance/api/graph.py b/tests/performance/api/graph.py new file mode 100644 index 00000000000..e2d2e7d2058 --- /dev/null +++ b/tests/performance/api/graph.py @@ -0,0 +1,105 @@ +# Apache License, Version 2.0 + +from . import TestQueue + +import json +import pathlib +from typing import Dict, List + +class TestGraph: + def __init__(self, json_filepaths: List[pathlib.Path]): + # Initialize graph from JSON file. Note that this is implemented without + # accessing any benchmark environment or configuration. This ways benchmarks + # run on various machines can be aggregated and the graph generated on another + # machine. + + # Gather entries for each device. + devices = {} + + for json_filepath in json_filepaths: + queue = TestQueue(json_filepath) + + for entry in queue.entries: + if entry.status in ('done', 'outdated'): + device_name = entry.device_name + if device_name in devices.keys(): + devices[device_name].append(entry) + else: + devices[device_name] = [entry] + + data = [] + for device_name, device_entries in devices.items(): + # Gather used categories. + categories = {} + for entry in device_entries: + category = entry.category + if category in categories.keys(): + categories[category].append(entry) + else: + categories[category] = [entry] + + # Generate one graph for every device x category combination. + for category, category_entries in categories.items(): + entries = sorted(category_entries, key=lambda entry: (entry.revision, entry.test)) + chart_type = 'line' if entries[0].benchmark_type == 'time_series' else 'comparison' + data.append(self.chart(device_name, category, entries, chart_type)) + + self.json = json.dumps(data, indent=2) + + def chart(self, device_name: str, category: str, entries: List, chart_type: str) -> Dict: + # Gather used tests. + tests = {} + for entry in entries: + test = entry.test + if test not in tests.keys(): + tests[test] = len(tests) + + # Gather used revisions. + revisions = {} + revision_dates = {} + for entry in entries: + revision = entry.revision + if revision not in revisions.keys(): + revisions[revision] = len(revisions) + revision_dates[revision] = int(entry.date) + + # Google Charts JSON data layout is like a spreadsheat table, with + # colums, rows and cells. We create one column for revision labels, + # and one column for each test. + cols = [] + if chart_type == 'line': + cols.append({'id': '', 'label': 'Date', 'type': 'date'}) + else: + cols.append({'id': '', 'label': 'Revision', 'type': 'string'}) + for test, test_index in tests.items(): + cols.append({'id': '', 'label': test, 'type': 'number'}) + + rows = [] + for revision, revision_index in revisions.items(): + if chart_type == 'line': + date = revision_dates[revision] + row = [{'f': None, 'v': 'Date({0})'.format(date * 1000)}] + else: + row = [{'f': None, 'v': revision}] + row += [{}] * len(tests) + rows.append({'c': row}) + + for entry in entries: + test_index = tests[entry.test] + revision_index = revisions[entry.revision] + time = entry.output['time'] + rows[revision_index]['c'][test_index + 1] = {'f': None, 'v': time} + + data = {'cols': cols, 'rows': rows} + return {'device': device_name, 'category': category, 'data': data, 'chart_type': chart_type} + + def write(self, filepath: pathlib.Path) -> None: + # Write HTML page with JSON graph data embedded. + template_dir = pathlib.Path(__file__).parent + with open(template_dir / 'graph.template.html', 'r') as f: + template = f.read() + + contents = template.replace('%JSON_DATA%', self.json) + with open(filepath, "w") as f: + f.write(contents) + diff --git a/tests/performance/api/graph.template.html b/tests/performance/api/graph.template.html new file mode 100644 index 00000000000..8929c2bdd1b --- /dev/null +++ b/tests/performance/api/graph.template.html @@ -0,0 +1,86 @@ + + + Benchmarks + + + + + + +

Benchmarks

+
+ ... +
+ + diff --git a/tests/performance/api/test.py b/tests/performance/api/test.py new file mode 100644 index 00000000000..23459b4b421 --- /dev/null +++ b/tests/performance/api/test.py @@ -0,0 +1,71 @@ +# Apache License, Version 2.0 + +import abc +import fnmatch +from typing import Dict, List + +class Test: + @abc.abstractmethod + def name(self) -> str: + """ + Name of the test. + """ + + @abc.abstractmethod + def category(self) -> str: + """ + Category of the test. + """ + + def use_device(self) -> bool: + """ + Test uses a specific CPU or GPU device. + """ + return False + + @abc.abstractmethod + def run(self, env, device_id: str) -> Dict: + """ + Execute the test and report results. + """ + +class TestCollection: + def __init__(self, env, names_filter: List=['*'], categories_filter: List=['*']): + import importlib + import pkgutil + import tests + + self.tests = [] + + # Find and import all Python files in the tests folder, and generate + # the list of tests for each. + for _, modname, _ in pkgutil.iter_modules(tests.__path__, 'tests.'): + module = importlib.import_module(modname) + tests = module.generate(env) + + for test in tests: + test_category = test.category() + found = False + for category_filter in categories_filter: + if fnmatch.fnmatch(test_category, category_filter): + found = True + if not found: + continue + + test_name = test.name() + found = False + for name_filter in names_filter: + if fnmatch.fnmatch(test_name, name_filter): + found = True + if not found: + continue + + self.tests.append(test) + + def find(self, test_name: str, test_category: str): + # Find a test based on name and category. + for test in self.tests: + if test.name() == test_name and test.category() == test_category: + return test + + return None diff --git a/tests/performance/benchmark b/tests/performance/benchmark new file mode 100755 index 00000000000..3b43bd0aa96 --- /dev/null +++ b/tests/performance/benchmark @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# Apache License, Version 2.0 + +import api +import argparse +import fnmatch +import pathlib +import shutil +import sys +import time +from typing import List + +def find_blender_git_dir() -> pathlib.Path: + # Find .git directory of the repository we are in. + cwd = pathlib.Path.cwd() + + for path in [cwd] + list(cwd.parents): + if (path / '.git').exists(): + return path + + return None + +def get_tests_base_dir(blender_git_dir: pathlib.Path) -> pathlib.Path: + # Benchmarks dir is next to the Blender source folder. + return blender_git_dir.parent / 'benchmark' + +def use_revision_columns(config: api.TestConfig) -> bool: + return config.benchmark_type == "comparison" and \ + len(config.queue.entries) > 0 and \ + not config.queue.has_multiple_revisions_to_build + +def print_header(config: api.TestConfig) -> None: + # Print header with revision columns headers. + if use_revision_columns(config): + header = "" + if config.queue.has_multiple_categories: + header += f"{'': <15} " + header += f"{'': <40} " + + for revision_name in config.revision_names(): + header += f"{revision_name: <20} " + print(header) + +def print_row(config: api.TestConfig, entries: List, end='\n') -> None: + # Print one or more test entries on a row. + row = "" + + # For time series, print revision first. + if not use_revision_columns(config): + revision = entries[0].revision + git_hash = entries[0].git_hash + + row += f"{revision: <15} " + + if config.queue.has_multiple_categories: + row += f"{entries[0].category: <15} " + row += f"{entries[0].test: <40} " + + for entry in entries: + # Show time or status. + status = entry.status + output = entry.output + result = '' + if status in ('done', 'outdated') and output: + result = '%.4fs' % output['time'] + + if status == 'outdated': + result += " (outdated)" + else: + result = status + + row += f"{result: <20} " + + print(row, end=end, flush=True) + + +def match_entry(entry: api.TestEntry, args: argparse.Namespace): + # Filter tests by name and category. + return fnmatch.fnmatch(entry.test, args.test) or \ + fnmatch.fnmatch(entry.category, args.test) or \ + entry.test.find(args.test) != -1 or \ + entry.category.find(args.test) != -1 + +def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry: api.TestEntry): + # Check if entry needs to be run. + if entry.status not in ('queued', 'outdated'): + print_row(config, row, end='\r') + return False + + # Run test entry. + revision = entry.revision + git_hash = entry.git_hash + testname = entry.test + testcategory = entry.category + device_type = entry.device_type + device_id = entry.device_id + + test = config.tests.find(testname, testcategory) + if not test: + return False + + # Log all output to dedicated log file. + logname = testcategory + '_' + testname + '_' + revision + if device_id != 'CPU': + logname += '_' + device_id + env.set_log_file(config.logs_dir / (logname + '.log'), clear=True) + + # Build revision, or just set path to existing executable. + entry.status = 'building' + print_row(config, row, end='\r') + if len(entry.executable): + env.set_blender_executable(pathlib.Path(entry.executable)) + else: + env.checkout(git_hash) + env.build(git_hash) + + # Run test and update output and status. + entry.status = 'running' + print_row(config, row, end='\r') + entry.output = test.run(env, device_id) + entry.status = 'done' if entry.output else 'failed' + print_row(config, row, end='\r') + + # Update device name in case the device changed since the entry was created. + entry.device_name = config.device_name(device_id) + + # Restore default logging and Blender executable. + env.unset_log_file() + env.unset_blender_executable() + + return True + +def cmd_init(env: api.TestEnvironment, argv: List): + # Initialize benchmarks folder. + parser = argparse.ArgumentParser() + parser.add_argument('--build', default=False, action='store_true') + args = parser.parse_args(argv) + env.set_log_file(env.base_dir / 'setup.log', clear=False) + env.init(args.build) + env.unset_log_file() + +def cmd_list(env: api.TestEnvironment, argv: List) -> None: + # List devices, tests and configurations. + print('DEVICES') + machine = env.get_machine() + for device in machine.devices: + name = f"{device.name} ({device.operating_system})" + print(f"{device.id: <15} {name}") + print('') + + print('TESTS') + collection = api.TestCollection(env) + for test in collection.tests: + print(f"{test.category(): <15} {test.name(): <50}") + print('') + + print('CONFIGS') + configs = env.get_configs(names_only=True) + for config_name in configs: + print(config_name) + +def cmd_status(env: api.TestEnvironment, argv: List): + # Print status of tests in configurations. + parser = argparse.ArgumentParser() + parser.add_argument('config', nargs='?', default=None) + parser.add_argument('test', nargs='?', default='*') + args = parser.parse_args(argv) + + configs = env.get_configs(args.config) + first = True + for config in configs: + if not args.config: + if first: + first = False + else: + print("") + print(config.name.upper()) + + print_header(config) + for row in config.queue.rows(use_revision_columns(config)): + if match_entry(row[0], args): + print_row(config, row) + +def cmd_reset(env: api.TestEnvironment, argv: List): + # Reset tests to re-run them. + parser = argparse.ArgumentParser() + parser.add_argument('config', nargs='?', default=None) + parser.add_argument('test', nargs='?', default='*') + args = parser.parse_args(argv) + + configs = env.get_configs(args.config) + for config in configs: + print_header(config) + for row in config.queue.rows(use_revision_columns(config)): + if match_entry(row[0], args): + for entry in row: + entry.status = 'queued' + entry.result = {} + print_row(config, row) + + config.queue.write() + +def cmd_run(env: api.TestEnvironment, argv: List): + # Run tests. + parser = argparse.ArgumentParser() + parser.add_argument('config', nargs='?', default=None) + parser.add_argument('test', nargs='?', default='*') + args = parser.parse_args(argv) + + configs = env.get_configs(args.config) + for config in configs: + updated = False + print_header(config) + for row in config.queue.rows(use_revision_columns(config)): + if match_entry(row[0], args): + for entry in row: + if run_entry(env, config, row, entry): + updated = True + # Write queue every time in case running gets interrupted, + # so it can be resumed. + config.queue.write() + print_row(config, row) + + if updated: + # Generate graph if test were run. + json_filepath = config.base_dir / "results.json" + html_filepath = config.base_dir / "results.html" + graph = api.TestGraph([json_filepath]) + graph.write(html_filepath) + + print("\nfile://" + str(html_filepath)) + +def cmd_graph(argv: List): + # Create graph from a given JSON results file. + parser = argparse.ArgumentParser() + parser.add_argument('json_file', nargs='+') + parser.add_argument('-o', '--output', type=str, required=True) + args = parser.parse_args(argv) + + graph = api.TestGraph([pathlib.Path(path) for path in args.json_file]) + graph.write(pathlib.Path(args.output)) + +def main(): + usage = ('benchmark []\n' + '\n' + 'Commands:\n' + ' init [--build] Init benchmarks directory and default config\n' + ' Optionally with automated revision building setup\n' + ' \n' + ' list List available tests, devices and configurations\n' + ' \n' + ' run [] [] Execute tests for configuration\n' + ' reset [] [] Clear tests results from config, for re-running\n' + ' status [] [] List configurations and their tests\n' + ' \n' + ' graph a.json b.json... -o out.html Create graph from results in JSON files\n') + + parser = argparse.ArgumentParser( + description='Blender performance testing', + usage=usage) + + parser.add_argument('command', nargs='?', default='help') + args = parser.parse_args(sys.argv[1:2]) + + argv = sys.argv[2:] + blender_git_dir = find_blender_git_dir() + if blender_git_dir == None: + sys.stderr.write('Error: no blender git repository found from current working directory\n') + sys.exit(1) + + if args.command == 'graph': + cmd_graph(argv) + sys.exit(0) + + base_dir = get_tests_base_dir(blender_git_dir) + env = api.TestEnvironment(blender_git_dir, base_dir) + if args.command == 'init': + cmd_init(env, argv) + sys.exit(0) + + if not env.base_dir.exists(): + sys.stderr.write('Error: benchmark directory not initialized\n') + sys.exit(1) + + if args.command == 'list': + cmd_list(env, argv) + elif args.command == 'run': + cmd_run(env, argv) + elif args.command == 'reset': + cmd_reset(env, argv) + elif args.command == 'status': + cmd_status(env, argv) + elif args.command == 'help': + parser.print_usage() + else: + sys.stderr.write(f'Unknown command: {args.command}\n') + +if __name__ == '__main__': + main() diff --git a/tests/performance/tests/__init__.py b/tests/performance/tests/__init__.py new file mode 100644 index 00000000000..69b72d23c3f --- /dev/null +++ b/tests/performance/tests/__init__.py @@ -0,0 +1,2 @@ +# Apache License, Version 2.0 + diff --git a/tests/performance/tests/animation.py b/tests/performance/tests/animation.py new file mode 100644 index 00000000000..de3b8817820 --- /dev/null +++ b/tests/performance/tests/animation.py @@ -0,0 +1,38 @@ +# Apache License, Version 2.0 + +import api +import os + +def _run(args): + import bpy + import time + + start_time = time.time() + + scene = bpy.context.scene + for i in range(scene.frame_start, scene.frame_end): + scene.frame_set(scene.frame_start) + + elapsed_time = time.time() - start_time + + result = {'time': elapsed_time} + return result + +class AnimationTest(api.Test): + def __init__(self, filepath): + self.filepath = filepath + + def name(self): + return self.filepath.stem + + def category(self): + return "animation" + + def run(self, env, device_id): + args = {} + result, _ = env.run_in_blender(_run, args) + return result + +def generate(env): + filepaths = env.find_blend_files('animation') + return [AnimationTest(filepath) for filepath in filepaths] diff --git a/tests/performance/tests/blend_load.py b/tests/performance/tests/blend_load.py new file mode 100644 index 00000000000..4df8bd774d3 --- /dev/null +++ b/tests/performance/tests/blend_load.py @@ -0,0 +1,39 @@ +# Apache License, Version 2.0 + +import api +import os +import pathlib + +def _run(filepath): + import bpy + import time + + # Load once to ensure it's cached by OS + bpy.ops.wm.open_mainfile(filepath=filepath) + bpy.ops.wm.read_homefile() + + # Measure loading the second time + start_time = time.time() + bpy.ops.wm.open_mainfile(filepath=filepath) + elapsed_time = time.time() - start_time + + result = {'time': elapsed_time} + return result + +class BlendLoadTest(api.Test): + def __init__(self, filepath): + self.filepath = filepath + + def name(self): + return self.filepath.stem + + def category(self): + return "blend_load" + + def run(self, env, device_id): + result, _ = env.run_in_blender(_run, str(self.filepath)) + return result + +def generate(env): + filepaths = env.find_blend_files('*/*') + return [BlendLoadTest(filepath) for filepath in filepaths] diff --git a/tests/performance/tests/cycles.py b/tests/performance/tests/cycles.py new file mode 100644 index 00000000000..185d4e8e6c0 --- /dev/null +++ b/tests/performance/tests/cycles.py @@ -0,0 +1,79 @@ +# Apache License, Version 2.0 + +import api +import os + +def _run(args): + import bpy + import time + + device_type = args['device_type'] + device_index = args['device_index'] + + scene = bpy.context.scene + scene.render.engine = 'CYCLES' + scene.render.filepath = args['render_filepath'] + scene.render.image_settings.file_format = 'PNG' + scene.cycles.device = 'CPU' if device_type == 'CPU' else 'GPU' + + if scene.cycles.device == 'GPU': + # Enable specified GPU in preferences. + prefs = bpy.context.preferences + cprefs = prefs.addons['cycles'].preferences + cprefs.compute_device_type = device_type + devices = cprefs.get_devices_for_type(device_type) + for device in devices: + device.use = False + + index = 0 + for device in devices: + if device.type == device_type: + if index == device_index: + device.use = True + break + else: + index += 1 + + # Render + bpy.ops.render.render(write_still=True) + + return None + +class CyclesTest(api.Test): + def __init__(self, filepath): + self.filepath = filepath + + def name(self): + return self.filepath.stem + + def category(self): + return "cycles" + + def use_device(self): + return True + + def run(self, env, device_id): + tokens = device_id.split('_') + device_type = tokens[0] + device_index = int(tokens[1]) if len(tokens) > 1 else 0 + args = {'device_type': device_type, + 'device_index': device_index, + 'render_filepath': str(env.log_file.parent / (env.log_file.stem + '.png'))} + + _, lines = env.run_in_blender(_run, args, ['--debug-cycles', '--verbose', '1', self.filepath]) + + # Parse render time from output + prefix = "Render time (without synchronization): " + time = 0.0 + for line in lines: + line = line.strip() + offset = line.find(prefix) + if offset != -1: + time = line[offset + len(prefix):] + return {'time': float(time)} + + raise Exception("Error parsing render time output") + +def generate(env): + filepaths = env.find_blend_files('cycles-x/*') + return [CyclesTest(filepath) for filepath in filepaths]