tmaq
/
tornavis
1
0
Fork 0
Code Issues 2 Pull Requests 29 Packages Projects 24 Releases Wiki Activity

TEST COMMIT: API doc generation changes. 

This commit is intended to be reverted within a few minutes.

commit 39ffb045a52d16994c1c87ccf3249ff3222a8fca
Author: Bastien Montagne <bastien@blender.org>
Date:   Wed Jun 15 15:43:13 2022 +0200

    Py API Doc: add runtime changelog generation to `sphinx_doc_gen.py`.

    Optionally use `sphinx_changelog_gen.py` to dump current version of the
    API in a JSON file, and use closest previous one listed in given index
    file to create a changelog RST page for Sphinx.

commit fbe354d3fcfa2ad1ed430c3c27e19b99a0266dda
Author: Bastien Montagne <bastien@blender.org>
Date:   Wed Jun 15 15:36:19 2022 +0200

    Py API Doc: refactor changelog generation script.

    Main change is to make it use JSON format for its dump files, instead of
    some Python code.

    It also introduces an index for those API dump files, mapping a blender
    version to the relevant file path.

    This is then used to automatically the most recent (version-number wise)
    previous API dump to compare against current one, when generating the
    change log RST file.
This commit is contained in:
Bastien Montagne 2022-06-15 16:48:30 +02:00
parent f0fa90e156
commit 298372fa06
2 changed files with 256 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

331
doc/python_api/sphinx_changelog_gen.py
View File

@ -1,59 +1,112 @@
# SPDX-License-Identifier: GPL-2.0-or-later # SPDX-License-Identifier: GPL-2.0-or-later
""" """
Dump the python API into a text file so we can generate changelogs. ---------------
output from this tool should be added into "doc/python_api/rst/change_log.rst" Dump the python API into a JSON file, or generate changelogs from those JSON API dumps.
# dump api blender_version.py in CWD Typically, changelog output from this tool should be added into "doc/python_api/rst/change_log.rst"
blender --background --python doc/python_api/sphinx_changelog_gen.py -- --dump
# create changelog API dump files are saved together with the generated API doc on the server, with a general index file.
This way the changelog generation simply needs to re-download the previous version's dump for the diffing process.
---------------
# Dump api blender_version.json in CWD:
blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \
--indexpath="path/to/api/docs/api_dump_index.json" \
dump --filepath-out="path/to/api/docs/<version>/api_dump.json"
# Create changelog:
blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \ blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \
--api_from blender_2_63_0.py \ --indexpath="path/to/api/docs/api_dump_index.json" \
--api_to blender_2_64_0.py \ changelog --filepath-out doc/python_api/rst/change_log.rst
--api_out changes.rst
# Api comparison can also run without blender,
# Api comparison can also run without blender # will by default generate changeloig between the last two available versions listed in the index,
# unless input files are provided explicitely:
python doc/python_api/sphinx_changelog_gen.py -- \ python doc/python_api/sphinx_changelog_gen.py -- \
--api_from blender_api_2_63_0.py \ --indexpath="path/to/api/docs/api_dump_index.json" \
--api_to blender_api_2_64_0.py \ changelog --filepath-in-from blender_api_2_63_0.json \
--api_out changes.rst --filepath-in-to blender_api_2_64_0.json \
--filepath-out changes.rst
# Save the latest API dump in this folder, renaming it with its revision. --------------
# This way the next person updating it doesn't need to build an old Blender only for that
API dump index format:
{[version_main, version_sub]: "<version>/api_dump.json", ...
}
API dump format:
[
[version_main, vserion_sub, version_path],
{"module.name":
{"parent.class":
{"basic_type", "member_name":
["Name", type, range, length, default, descr, f_args, f_arg_types, f_ret_types]}, ...
}, ...
}
]
""" """
# format import json
''' import os
{"module.name":
{"parent.class":
{"basic_type", "member_name":
("Name", type, range, length, default, descr, f_args, f_arg_types, f_ret_types)}, ...
}, ...
}
'''
api_names = "basic_type" "name", "type", "range", "length", "default", "descr", "f_args", "f_arg_types", "f_ret_types" api_names = "basic_type" "name", "type", "range", "length", "default", "descr", "f_args", "f_arg_types", "f_ret_types"
API_BASIC_TYPE = 0 API_BASIC_TYPE = 0
API_F_ARGS = 7 API_F_ARGS = 7
def api_dunp_fname(): def api_version():
import bpy try:
return "blender_api_%s.py" % "_".join([str(i) for i in bpy.app.version]) import bpy
except:
return None, None
version = tuple(bpy.app.version[:2])
version_key = "%d.%d" % (version[0], version[1])
return version, version_key
def api_dump(): def api_version_previous_in_index(index, version):
dump = {} print("Searching for previous version to %s in %r" % (version, index))
dump_module = dump["bpy.types"] = {} version_prev = (version[0], version[1])
while True:
version_prev = (version_prev[0], version_prev[1] - 1)
if version_prev[1] < 0:
version_prev = (version_prev[0] - 1, 99)
if version_prev[0] < 0:
return None, None
version_prev_key = "%d.%d" % (version_prev[0], version_prev[1])
print("Checking for previous version %s" % (version_prev,))
if version_prev_key in index:
print("Found previous version %s: %r" % (version_prev, index[version_prev_key]))
return version_prev, version_prev_key
class JSONEncoderAPIDump(json.JSONEncoder):
def default(self, o):
if o is ...:
return "..."
if isinstance(o, set):
return tuple(o)
return json.JSONEncoder.default(self, o)
def api_dump(args):
import rna_info import rna_info
import inspect import inspect
version, version_key = api_version()
if version is None:
raise(ValueError("API dumps can only be generated from within Blender."))
dump = {}
dump_module = dump["bpy.types"] = {}
struct = rna_info.BuildRNAInfo()[0] struct = rna_info.BuildRNAInfo()[0]
for struct_id, struct_info in sorted(struct.items()): for struct_id, struct_info in sorted(struct.items()):
@ -155,17 +208,24 @@ def api_dump():
) )
del funcs del funcs
import pprint filepath_out = args.filepath_out
with open(filepath_out, 'w', encoding='utf-8') as file_handle:
json.dump((version, dump), file_handle, cls=JSONEncoderAPIDump)
filename = api_dunp_fname() indexpath = args.indexpath
filehandle = open(filename, 'w', encoding='utf-8') if os.path.exists(indexpath):
tot = filehandle.write(pprint.pformat(dump, width=1)) with open(indexpath, 'r', encoding='utf-8') as file_handle:
filehandle.close() index = json.load(file_handle)
print("%s, %d bytes written" % (filename, tot)) else:
index = {}
index[version_key] = filepath_out
with open(indexpath, 'w', encoding='utf-8') as file_handle:
json.dump(index, file_handle)
print("API version %s dumped into %r, and index %r has been updated" % (version_key, filepath_out, indexpath))
def compare_props(a, b, fuzz=0.75): def compare_props(a, b, fuzz=0.75):
# must be same basic_type, function != property # must be same basic_type, function != property
if a[0] != b[0]: if a[0] != b[0]:
return False return False
@ -180,15 +240,45 @@ def compare_props(a, b, fuzz=0.75):
return ((tot / totlen) >= fuzz) return ((tot / totlen) >= fuzz)
def api_changelog(api_from, api_to, api_out): def api_changelog(args):
indexpath = args.indexpath
filepath_in_from = args.filepath_in_from
filepath_in_to = args.filepath_in_to
filepath_out = args.filepath_out
file_handle = open(api_from, 'r', encoding='utf-8') rootpath = os.path.dirname(indexpath)
dict_from = eval(file_handle.read())
file_handle.close()
file_handle = open(api_to, 'r', encoding='utf-8') version, version_key = api_version()
dict_to = eval(file_handle.read()) if version is None and (filepath_in_from is None or filepath_in_to is None):
file_handle.close() raise(ValueError("API dumps files must be given when ran outside of Blender."))
with open(indexpath, 'r', encoding='utf-8') as file_handle:
index = json.load(file_handle)
if filepath_in_to == None:
filepath_in_to = index.get(version_key, None)
if filepath_in_to == None:
raise(ValueError("Cannot find API dump file for Blender version " + str(version) + " in index file."))
print("Found to file: %r" % filepath_in_to)
if filepath_in_from == None:
version_from, version_from_key = api_version_previous_in_index(index, version)
if version_from is None:
raise(ValueError("No previous version of Blender could be found in the index."))
filepath_in_from = index.get(version_from_key, None)
if filepath_in_from is None:
raise(ValueError("Cannot find API dump file for previous Blender version " + str(version_from) + " in index file."))
print("Found from file: %r" % filepath_in_from)
with open(os.path.join(rootpath, filepath_in_from), 'r', encoding='utf-8') as file_handle:
_, dict_from = json.load(file_handle)
with open(os.path.join(rootpath, filepath_in_to), 'r', encoding='utf-8') as file_handle:
dump_version, dict_to = json.load(file_handle)
assert(tuple(dump_version) == version)
api_changes = [] api_changes = []
@ -249,63 +339,69 @@ def api_changelog(api_from, api_to, api_out):
# also document function argument changes # also document function argument changes
fout = open(api_out, 'w', encoding='utf-8') with open(filepath_out, 'w', encoding='utf-8') as fout:
fw = fout.write fw = fout.write
# print(api_changes) # print(api_changes)
# :class:`bpy_struct.id_data` # :class:`bpy_struct.id_data`
def write_title(title, title_char): # Write header.
fw("%s\n%s\n\n" % (title, title_char * len(title))) fw(""
":tocdepth: 2\n"
"\n"
"Blender API Change Log\n"
"**********************\n"
"\n"
".. note, this document is auto generated by sphinx_changelog_gen.py\n"
"\n"
"\n"
"%s to %s\n"
"============\n"
"\n" % (version_from_key, version_key))
for mod_id, class_id, props_moved, props_new, props_old, func_args in api_changes: def write_title(title, title_char):
class_name = class_id.split(".")[-1] fw("%s\n%s\n\n" % (title, title_char * len(title)))
title = mod_id + "." + class_name
write_title(title, "-")
if props_new: for mod_id, class_id, props_moved, props_new, props_old, func_args in api_changes:
write_title("Added", "^") class_name = class_id.split(".")[-1]
for prop_id in props_new: title = mod_id + "." + class_name
fw("* :class:`%s.%s.%s`\n" % (mod_id, class_name, prop_id)) write_title(title, "-")
fw("\n")
if props_old: if props_new:
write_title("Removed", "^") write_title("Added", "^")
for prop_id in props_old: for prop_id in props_new:
fw("* **%s**\n" % prop_id) # can't link to removed docs fw("* :class:`%s.%s.%s`\n" % (mod_id, class_name, prop_id))
fw("\n") fw("\n")
if props_moved: if props_old:
write_title("Renamed", "^") write_title("Removed", "^")
for prop_id_old, prop_id in props_moved: for prop_id in props_old:
fw("* **%s** -> :class:`%s.%s.%s`\n" % (prop_id_old, mod_id, class_name, prop_id)) fw("* **%s**\n" % prop_id) # can't link to removed docs
fw("\n") fw("\n")
if func_args: if props_moved:
write_title("Function Arguments", "^") write_title("Renamed", "^")
for func_id, args_old, args_new in func_args: for prop_id_old, prop_id in props_moved:
args_new = ", ".join(args_new) fw("* **%s** -> :class:`%s.%s.%s`\n" % (prop_id_old, mod_id, class_name, prop_id))
args_old = ", ".join(args_old) fw("\n")
fw("* :class:`%s.%s.%s` (%s), *was (%s)*\n" % (mod_id, class_name, func_id, args_new, args_old))
fw("\n")
fout.close() if func_args:
write_title("Function Arguments", "^")
for func_id, args_old, args_new in func_args:
args_new = ", ".join(args_new)
args_old = ", ".join(args_old)
fw("* :class:`%s.%s.%s` (%s), *was (%s)*\n" % (mod_id, class_name, func_id, args_new, args_old))
fw("\n")
print("Written: %r" % api_out) print("Written: %r" % filepath_out)
def main(): def main(argv=None):
import sys import sys
import os import argparse
try: if argv is None:
import argparse argv = sys.argv
except ImportError:
print("Old Blender, just dumping")
api_dump()
return
argv = sys.argv
if "--" not in argv: if "--" not in argv:
argv = [] # as if no args are passed argv = [] # as if no args are passed
@ -316,42 +412,39 @@ def main():
usage_text = "Run blender in background mode with this script: " usage_text = "Run blender in background mode with this script: "
"blender --background --factory-startup --python %s -- [options]" % os.path.basename(__file__) "blender --background --factory-startup --python %s -- [options]" % os.path.basename(__file__)
epilog = "Run this before releases" parser = argparse.ArgumentParser(description=usage_text,
epilog=__doc__,
parser = argparse.ArgumentParser(description=usage_text, epilog=epilog) formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument( parser.add_argument(
"--dump", dest="dump", action='store_true', "--indexpath", dest="indexpath", metavar='FILE', required=True,
help="When set the api will be dumped into blender_version.py") help="Path of the JSON file containing the index of all available API dumps.")
parser.add_argument( parser_commands = parser.add_subparsers(required=True)
"--api_from", dest="api_from", metavar='FILE',
help="File to compare from (previous version)")
parser.add_argument(
"--api_to", dest="api_to", metavar='FILE',
help="File to compare from (current)")
parser.add_argument(
"--api_out", dest="api_out", metavar='FILE',
help="Output sphinx changelog")
args = parser.parse_args(argv) # In this example we won't use the args parser_dump = parser_commands.add_parser('dump', help="Dump the current Blender Python API into a JSON file.")
parser_dump.add_argument(
"--filepath-out", dest="filepath_out", metavar='FILE', required=True,
help="Path of the JSON file containing the dump of the API.")
parser_dump.set_defaults(func=api_dump)
if not argv: parser_changelog = parser_commands.add_parser('changelog', help="Generate the RST changelog page based on two Blender Python API JSON dumps.")
print("No args given!")
parser.print_help()
return
if args.dump: parser_changelog.add_argument(
api_dump() "--filepath-in-from", dest="filepath_in_from", metavar='FILE', default=None,
else: help="JSON dump file to compare from (typically, previous version). "
if args.api_from and args.api_to and args.api_out: "If not given, will be automatically determined from current Blender version and index file.")
api_changelog(args.api_from, args.api_to, args.api_out) parser_changelog.add_argument(
else: "--filepath-in-to", dest="filepath_in_to", metavar='FILE', default=None,
print("Error: --api_from/api_to/api_out args needed") help="JSON dump file to compare to (typically, current version). "
parser.print_help() "If not given, will be automatically determined from current Blender version and index file.")
return parser_changelog.add_argument(
"--filepath-out", dest="filepath_out", metavar='FILE', required=True,
help="Output sphinx changelog RST file.")
parser_changelog.set_defaults(func=api_changelog)
print("batch job finished, exiting") args = parser.parse_args(argv)
args.func(args)
if __name__ == "__main__": if __name__ == "__main__":

44
doc/python_api/sphinx_doc_gen.py
View File

@ -141,6 +141,26 @@ def handle_args():
required=False, required=False,
) )
parser.add_argument(
"--api-changelog-generate",
dest="changelog",
default=False,
action='store_true',
help="Generate the API changelog RST file "
"(default=False, requires `--api-dump-index-path` parameter)",
required=False,
)
parser.add_argument(
"--api-dump-index-path",
dest="api_dump_index_path",
metavar='FILE',
default=None,
help="Path to the API dump index JSON file "
"(required when `--api-changelog-generate` is True)",
required=False,
)
parser.add_argument( parser.add_argument(
"-o", "--output", "-o", "--output",
dest="output_dir", dest="output_dir",
@ -514,6 +534,27 @@ if ARGS.sphinx_build_pdf:
sphinx_make_pdf_log = os.path.join(ARGS.output_dir, ".latex_make.log") sphinx_make_pdf_log = os.path.join(ARGS.output_dir, ".latex_make.log")
SPHINX_MAKE_PDF_STDOUT = open(sphinx_make_pdf_log, "w", encoding="utf-8") SPHINX_MAKE_PDF_STDOUT = open(sphinx_make_pdf_log, "w", encoding="utf-8")
# --------------------------------CHANGELOG GENERATION--------------------------------------
API_DUMP_INDEX_FILEPATH = ARGS.api_dump_index_path
API_DUMP_ROOT = os.path.dirname(API_DUMP_INDEX_FILEPATH)
API_DUMP_FILEPATH = os.path.abspath(os.path.join(API_DUMP_ROOT, BLENDER_VERSION_DOTS, "api_dump.json"))
API_CHANGELOG_FILEPATH = os.path.abspath(os.path.join(SPHINX_IN_TMP, "change_log.rst"))
def generate_changelog():
import importlib.util
spec = importlib.util.spec_from_file_location("sphinx_changelog_gen",
os.path.abspath(os.path.join(SCRIPT_DIR, "sphinx_changelog_gen.py")))
sphinx_changelog_gen = importlib.util.module_from_spec(spec)
spec.loader.exec_module(sphinx_changelog_gen)
sphinx_changelog_gen.main(("--", "--indexpath", API_DUMP_INDEX_FILEPATH, "dump", "--filepath-out", API_DUMP_FILEPATH))
sphinx_changelog_gen.main(("--", "--indexpath", API_DUMP_INDEX_FILEPATH, "changelog", "--filepath-out", API_CHANGELOG_FILEPATH))
# --------------------------------API DUMP-------------------------------------- # --------------------------------API DUMP--------------------------------------
# Lame, python won't give some access. # Lame, python won't give some access.
@ -2473,6 +2514,9 @@ def main():
rna2sphinx(SPHINX_IN_TMP) rna2sphinx(SPHINX_IN_TMP)
if ARGS.changelog:
generate_changelog()
if ARGS.full_rebuild: if ARGS.full_rebuild:
# Only for full updates. # Only for full updates.
shutil.rmtree(SPHINX_IN, True) shutil.rmtree(SPHINX_IN, True)