diff --git a/docs/mindscience/docs/_ext/customdocumenter.txt b/docs/mindscience/docs/_ext/customdocumenter.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d37ae41f6772a21da2a7dc5c7bff75128e68330 --- /dev/null +++ b/docs/mindscience/docs/_ext/customdocumenter.txt @@ -0,0 +1,245 @@ +import re +import os +from sphinx.ext.autodoc import Documenter + + +class CustomDocumenter(Documenter): + + def document_members(self, all_members: bool = False) -> None: + """Generate reST for member documentation. + + If *all_members* is True, do all members, else those given by + *self.options.members*. + """ + # set current namespace for finding members + self.env.temp_data['autodoc:module'] = self.modname + if self.objpath: + self.env.temp_data['autodoc:class'] = self.objpath[0] + + want_all = all_members or self.options.inherited_members or \ + self.options.members is ALL + # find out which members are documentable + members_check_module, members = self.get_object_members(want_all) + + # **** 排除已写中文接口名 **** + file_path = os.path.join(self.env.app.srcdir, self.env.docname+'.rst') + exclude_re = re.compile(r'(.. py:class::|.. py:function::)\s+(.*?)(\(|\n)') + includerst_re = re.compile(r'.. include::\s+(.*?)\n') + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + excluded_members = exclude_re.findall(content) + if excluded_members: + excluded_members = [i[1].split('.')[-1] for i in excluded_members] + rst_included = includerst_re.findall(content) + if rst_included: + for i in rst_included: + include_path = os.path.join(os.path.dirname(file_path), i) + if os.path.exists(include_path): + with open(include_path, 'r', encoding='utf8') as g: + content_ = g.read() + excluded_member_ = exclude_re.findall(content_) + if excluded_member_: + excluded_member_ = [j[1].split('.')[-1] for j in excluded_member_] + excluded_members.extend(excluded_member_) + + if excluded_members: + if self.options.exclude_members: + self.options.exclude_members |= set(excluded_members) + else: + self.options.exclude_members = excluded_members + + # remove members given by exclude-members + if self.options.exclude_members: + members = [ + (membername, member) for (membername, member) in members + if ( + self.options.exclude_members is ALL or + membername not in self.options.exclude_members + ) + ] + + # document non-skipped members + memberdocumenters = [] # type: List[Tuple[Documenter, bool]] + for (mname, member, isattr) in self.filter_members(members, want_all): + classes = [cls for cls in self.documenters.values() + if cls.can_document_member(member, mname, isattr, self)] + if not classes: + # don't know how to document this member + continue + # prefer the documenter with the highest priority + classes.sort(key=lambda cls: cls.priority) + # give explicitly separated module name, so that members + # of inner classes can be documented + full_mname = self.modname + '::' + \ + '.'.join(self.objpath + [mname]) + documenter = classes[-1](self.directive, full_mname, self.indent) + memberdocumenters.append((documenter, isattr)) + member_order = self.options.member_order or \ + self.env.config.autodoc_member_order + if member_order == 'groupwise': + # sort by group; relies on stable sort to keep items in the + # same group sorted alphabetically + memberdocumenters.sort(key=lambda e: e[0].member_order) + elif member_order == 'bysource' and self.analyzer: + # sort by source order, by virtue of the module analyzer + tagorder = self.analyzer.tagorder + + def keyfunc(entry: Tuple[Documenter, bool]) -> int: + fullname = entry[0].name.split('::')[1] + return tagorder.get(fullname, len(tagorder)) + memberdocumenters.sort(key=keyfunc) + + for documenter, isattr in memberdocumenters: + documenter.generate( + all_members=True, real_modname=self.real_modname, + check_module=members_check_module and not isattr) + + # reset current objects + self.env.temp_data['autodoc:module'] = None + self.env.temp_data['autodoc:class'] = None + + def generate(self, more_content: Any = None, real_modname: str = None, + check_module: bool = False, all_members: bool = False) -> None: + """Generate reST for the object given by *self.name*, and possibly for + its members. + + If *more_content* is given, include that content. If *real_modname* is + given, use that module name to find attribute docs. If *check_module* is + True, only generate if the object is defined in the module name it is + imported from. If *all_members* is True, document all members. + """ + if not self.parse_name(): + # need a module to import + logger.warning( + __('don\'t know which module to import for autodocumenting ' + '%r (try placing a "module" or "currentmodule" directive ' + 'in the document, or giving an explicit module name)') % + self.name, type='autodoc') + return + + # now, import the module and get object to document + if not self.import_object(): + return + + # If there is no real module defined, figure out which to use. + # The real module is used in the module analyzer to look up the module + # where the attribute documentation would actually be found in. + # This is used for situations where you have a module that collects the + # functions and classes of internal submodules. + self.real_modname = real_modname or self.get_real_modname() # type: str + + # try to also get a source code analyzer for attribute docs + try: + self.analyzer = ModuleAnalyzer.for_module(self.real_modname) + # parse right now, to get PycodeErrors on parsing (results will + # be cached anyway) + self.analyzer.find_attr_docs() + except PycodeError as err: + logger.debug('[autodoc] module analyzer failed: %s', err) + # no source file -- e.g. for builtin and C modules + self.analyzer = None + # at least add the module.__file__ as a dependency + if hasattr(self.module, '__file__') and self.module.__file__: + self.directive.filename_set.add(self.module.__file__) + else: + self.directive.filename_set.add(self.analyzer.srcname) + + # check __module__ of object (for members not given explicitly) + if check_module: + if not self.check_module(): + return + + # document members, if possible + self.document_members(all_members) + + +class ModuleDocumenter(CustomDocumenter): + """ + Specialized Documenter subclass for modules. + """ + objtype = 'module' + content_indent = '' + titles_allowed = True + + option_spec = { + 'members': members_option, 'undoc-members': bool_option, + 'noindex': bool_option, 'inherited-members': bool_option, + 'show-inheritance': bool_option, 'synopsis': identity, + 'platform': identity, 'deprecated': bool_option, + 'member-order': identity, 'exclude-members': members_set_option, + 'private-members': bool_option, 'special-members': members_option, + 'imported-members': bool_option, 'ignore-module-all': bool_option + } # type: Dict[str, Callable] + + def __init__(self, *args: Any) -> None: + super().__init__(*args) + merge_members_option(self.options) + + @classmethod + def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any + ) -> bool: + # don't document submodules automatically + return False + + def resolve_name(self, modname: str, parents: Any, path: str, base: Any + ) -> Tuple[str, List[str]]: + if modname is not None: + logger.warning(__('"::" in automodule name doesn\'t make sense'), + type='autodoc') + return (path or '') + base, [] + + def parse_name(self) -> bool: + ret = super().parse_name() + if self.args or self.retann: + logger.warning(__('signature arguments or return annotation ' + 'given for automodule %s') % self.fullname, + type='autodoc') + return ret + + def add_directive_header(self, sig: str) -> None: + Documenter.add_directive_header(self, sig) + + sourcename = self.get_sourcename() + + # add some module-specific options + if self.options.synopsis: + self.add_line(' :synopsis: ' + self.options.synopsis, sourcename) + if self.options.platform: + self.add_line(' :platform: ' + self.options.platform, sourcename) + if self.options.deprecated: + self.add_line(' :deprecated:', sourcename) + + def get_object_members(self, want_all: bool) -> Tuple[bool, List[Tuple[str, object]]]: + if want_all: + if (self.options.ignore_module_all or not + hasattr(self.object, '__all__')): + # for implicit module members, check __module__ to avoid + # documenting imported objects + return True, get_module_members(self.object) + else: + memberlist = self.object.__all__ + # Sometimes __all__ is broken... + if not isinstance(memberlist, (list, tuple)) or not \ + all(isinstance(entry, str) for entry in memberlist): + logger.warning( + __('__all__ should be a list of strings, not %r ' + '(in module %s) -- ignoring __all__') % + (memberlist, self.fullname), + type='autodoc' + ) + # fall back to all members + return True, get_module_members(self.object) + else: + memberlist = self.options.members or [] + ret = [] + for mname in memberlist: + try: + ret.append((mname, safe_getattr(self.object, mname))) + except AttributeError: + logger.warning( + __('missing attribute mentioned in :members: or __all__: ' + 'module %s, attribute %s') % + (safe_getattr(self.object, '__name__', '???'), mname), + type='autodoc' + ) + return False, ret diff --git a/docs/mindscience/docs/_ext/overwriteautosummary_generate.txt b/docs/mindscience/docs/_ext/overwriteautosummary_generate.txt new file mode 100644 index 0000000000000000000000000000000000000000..4b0a1b1dd2b410ecab971b13da9993c90d65ef0d --- /dev/null +++ b/docs/mindscience/docs/_ext/overwriteautosummary_generate.txt @@ -0,0 +1,707 @@ +""" + sphinx.ext.autosummary.generate + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Usable as a library or script to generate automatic RST source files for + items referred to in autosummary:: directives. + + Each generated RST file contains a single auto*:: directive which + extracts the docstring of the referred item. + + Example Makefile rule:: + + generate: + sphinx-autogen -o source/generated source/*.rst + + :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import argparse +import importlib +import inspect +import locale +import os +import pkgutil +import pydoc +import re +import sys +import warnings +from gettext import NullTranslations +from os import path +from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, Type, Union + +from jinja2 import TemplateNotFound +from jinja2.sandbox import SandboxedEnvironment + +import sphinx.locale +from sphinx import __display_version__, package_dir +from sphinx.application import Sphinx +from sphinx.builders import Builder +from sphinx.config import Config +from sphinx.deprecation import RemovedInSphinx50Warning +from sphinx.ext.autodoc import Documenter +from sphinx.ext.autodoc.importer import import_module +from sphinx.ext.autosummary import (ImportExceptionGroup, get_documenter, import_by_name, + import_ivar_by_name) +from sphinx.locale import __ +from sphinx.pycode import ModuleAnalyzer, PycodeError +from sphinx.registry import SphinxComponentRegistry +from sphinx.util import logging, rst, split_full_qualified_name, get_full_modname +from sphinx.util.inspect import getall, safe_getattr +from sphinx.util.osutil import ensuredir +from sphinx.util.template import SphinxTemplateLoader + +logger = logging.getLogger(__name__) + + +class DummyApplication: + """Dummy Application class for sphinx-autogen command.""" + + def __init__(self, translator: NullTranslations) -> None: + self.config = Config() + self.registry = SphinxComponentRegistry() + self.messagelog: List[str] = [] + self.srcdir = "/" + self.translator = translator + self.verbosity = 0 + self._warncount = 0 + self.warningiserror = False + + self.config.add('autosummary_context', {}, True, None) + self.config.add('autosummary_filename_map', {}, True, None) + self.config.add('autosummary_ignore_module_all', True, 'env', bool) + self.config.add('docs_branch', '', True, None) + self.config.add('branch', '', True, None) + self.config.add('cst_module_name', '', True, None) + self.config.add('copy_repo', '', True, None) + self.config.add('giturl', '', True, None) + self.config.add('repo_whl', '', True, None) + self.config.init_values() + + def emit_firstresult(self, *args: Any) -> None: + pass + + +class AutosummaryEntry(NamedTuple): + name: str + path: str + template: str + recursive: bool + + +def setup_documenters(app: Any) -> None: + from sphinx.ext.autodoc import (AttributeDocumenter, ClassDocumenter, DataDocumenter, + DecoratorDocumenter, ExceptionDocumenter, + FunctionDocumenter, MethodDocumenter, ModuleDocumenter, + NewTypeAttributeDocumenter, NewTypeDataDocumenter, + PropertyDocumenter) + documenters: List[Type[Documenter]] = [ + ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter, + FunctionDocumenter, MethodDocumenter, NewTypeAttributeDocumenter, + NewTypeDataDocumenter, AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter, + ] + for documenter in documenters: + app.registry.add_documenter(documenter.objtype, documenter) + + +def _simple_info(msg: str) -> None: + warnings.warn('_simple_info() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + print(msg) + + +def _simple_warn(msg: str) -> None: + warnings.warn('_simple_warn() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + print('WARNING: ' + msg, file=sys.stderr) + + +def _underline(title: str, line: str = '=') -> str: + if '\n' in title: + raise ValueError('Can only underline single lines') + return title + '\n' + line * len(title) + + +class AutosummaryRenderer: + """A helper class for rendering.""" + + def __init__(self, app: Union[Builder, Sphinx], template_dir: str = None) -> None: + if isinstance(app, Builder): + warnings.warn('The first argument for AutosummaryRenderer has been ' + 'changed to Sphinx object', + RemovedInSphinx50Warning, stacklevel=2) + if template_dir: + warnings.warn('template_dir argument for AutosummaryRenderer is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + + system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')] + loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path, + system_templates_path) + + self.env = SandboxedEnvironment(loader=loader) + self.env.filters['escape'] = rst.escape + self.env.filters['e'] = rst.escape + self.env.filters['underline'] = _underline + + if isinstance(app, (Sphinx, DummyApplication)): + if app.translator: + self.env.add_extension("jinja2.ext.i18n") + self.env.install_gettext_translations(app.translator) + elif isinstance(app, Builder): + if app.app.translator: + self.env.add_extension("jinja2.ext.i18n") + self.env.install_gettext_translations(app.app.translator) + + def exists(self, template_name: str) -> bool: + """Check if template file exists.""" + warnings.warn('AutosummaryRenderer.exists() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + try: + self.env.get_template(template_name) + return True + except TemplateNotFound: + return False + + def render(self, template_name: str, context: Dict) -> str: + """Render a template file.""" + try: + template = self.env.get_template(template_name) + except TemplateNotFound: + try: + # objtype is given as template_name + template = self.env.get_template('autosummary/%s.rst' % template_name) + except TemplateNotFound: + # fallback to base.rst + template = self.env.get_template('autosummary/base.rst') + + return template.render(context) + + +# -- Generating output --------------------------------------------------------- + + +class ModuleScanner: + def __init__(self, app: Any, obj: Any) -> None: + self.app = app + self.object = obj + + def get_object_type(self, name: str, value: Any) -> str: + return get_documenter(self.app, value, self.object).objtype + + def is_skipped(self, name: str, value: Any, objtype: str) -> bool: + try: + return self.app.emit_firstresult('autodoc-skip-member', objtype, + name, value, False, {}) + except Exception as exc: + logger.warning(__('autosummary: failed to determine %r to be documented, ' + 'the following exception was raised:\n%s'), + name, exc, type='autosummary') + return False + + def scan(self, imported_members: bool) -> List[str]: + members = [] + for name in members_of(self.object, self.app.config): + try: + value = safe_getattr(self.object, name) + except AttributeError: + value = None + + objtype = self.get_object_type(name, value) + if self.is_skipped(name, value, objtype): + continue + + try: + if inspect.ismodule(value): + imported = True + elif safe_getattr(value, '__module__') != self.object.__name__: + imported = True + else: + imported = False + except AttributeError: + imported = False + + respect_module_all = not self.app.config.autosummary_ignore_module_all + if imported_members: + # list all members up + members.append(name) + elif imported is False: + # list not-imported members + members.append(name) + elif '__all__' in dir(self.object) and respect_module_all: + # list members that have __all__ set + members.append(name) + + return members + + +def members_of(obj: Any, conf: Config) -> Sequence[str]: + """Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute + + Follows the ``conf.autosummary_ignore_module_all`` setting.""" + + if conf.autosummary_ignore_module_all: + return dir(obj) + else: + return getall(obj) or dir(obj) + + +def generate_autosummary_content(name: str, obj: Any, parent: Any, + template: AutosummaryRenderer, template_name: str, + imported_members: bool, app: Any, + recursive: bool, context: Dict, + modname: str = None, qualname: str = None) -> str: + doc = get_documenter(app, obj, parent) + + def skip_member(obj: Any, name: str, objtype: str) -> bool: + try: + return app.emit_firstresult('autodoc-skip-member', objtype, name, + obj, False, {}) + except Exception as exc: + logger.warning(__('autosummary: failed to determine %r to be documented, ' + 'the following exception was raised:\n%s'), + name, exc, type='autosummary') + return False + + def get_class_members(obj: Any) -> Dict[str, Any]: + members = sphinx.ext.autodoc.get_class_members(obj, [qualname], safe_getattr) + return {name: member.object for name, member in members.items()} + + def get_module_members(obj: Any) -> Dict[str, Any]: + members = {} + for name in members_of(obj, app.config): + try: + members[name] = safe_getattr(obj, name) + except AttributeError: + continue + return members + + def get_all_members(obj: Any) -> Dict[str, Any]: + if doc.objtype == "module": + return get_module_members(obj) + elif doc.objtype == "class": + return get_class_members(obj) + return {} + + def get_members(obj: Any, types: Set[str], include_public: List[str] = [], + imported: bool = True) -> Tuple[List[str], List[str]]: + items: List[str] = [] + public: List[str] = [] + + all_members = get_all_members(obj) + for name, value in all_members.items(): + documenter = get_documenter(app, value, obj) + if documenter.objtype in types: + # skip imported members if expected + if imported or getattr(value, '__module__', None) == obj.__name__: + skipped = skip_member(value, name, documenter.objtype) + if skipped is True: + pass + elif skipped is False: + # show the member forcedly + items.append(name) + public.append(name) + else: + items.append(name) + if name in include_public or not name.startswith('_'): + # considers member as public + public.append(name) + return public, items + + def get_module_attrs(members: Any) -> Tuple[List[str], List[str]]: + """Find module attributes with docstrings.""" + attrs, public = [], [] + try: + analyzer = ModuleAnalyzer.for_module(name) + attr_docs = analyzer.find_attr_docs() + for namespace, attr_name in attr_docs: + if namespace == '' and attr_name in members: + attrs.append(attr_name) + if not attr_name.startswith('_'): + public.append(attr_name) + except PycodeError: + pass # give up if ModuleAnalyzer fails to parse code + return public, attrs + + def get_modules(obj: Any) -> Tuple[List[str], List[str]]: + items: List[str] = [] + for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__): + fullname = name + '.' + modname + try: + module = import_module(fullname) + if module and hasattr(module, '__sphinx_mock__'): + continue + except ImportError: + pass + + items.append(fullname) + public = [x for x in items if not x.split('.')[-1].startswith('_')] + return public, items + + ns: Dict[str, Any] = {} + ns.update(context) + + if doc.objtype == 'module': + scanner = ModuleScanner(app, obj) + ns['members'] = scanner.scan(imported_members) + ns['functions'], ns['all_functions'] = \ + get_members(obj, {'function'}, imported=imported_members) + ns['classes'], ns['all_classes'] = \ + get_members(obj, {'class'}, imported=imported_members) + ns['exceptions'], ns['all_exceptions'] = \ + get_members(obj, {'exception'}, imported=imported_members) + ns['attributes'], ns['all_attributes'] = \ + get_module_attrs(ns['members']) + ispackage = hasattr(obj, '__path__') + if ispackage and recursive: + ns['modules'], ns['all_modules'] = get_modules(obj) + elif doc.objtype == 'class': + ns['members'] = dir(obj) + ns['inherited_members'] = \ + set(dir(obj)) - set(obj.__dict__.keys()) + ns['methods'], ns['all_methods'] = \ + get_members(obj, {'method'}, ['__init__']) + ns['attributes'], ns['all_attributes'] = \ + get_members(obj, {'attribute', 'property'}) + + if modname is None or qualname is None: + modname, qualname = split_full_qualified_name(name) + + if doc.objtype in ('method', 'attribute', 'property'): + ns['class'] = qualname.rsplit(".", 1)[0] + + if doc.objtype in ('class',): + shortname = qualname + else: + shortname = qualname.rsplit(".", 1)[-1] + + ns['fullname'] = name + ns['module'] = modname + ns['objname'] = qualname + ns['name'] = shortname + + ns['objtype'] = doc.objtype + ns['underline'] = len(name) * '=' + + if template_name: + return template.render(template_name, ns) + else: + return template.render(doc.objtype, ns) + + +def generate_autosummary_docs(sources: List[str], output_dir: str = None, + suffix: str = '.rst', base_path: str = None, + builder: Builder = None, template_dir: str = None, + imported_members: bool = False, app: Any = None, + overwrite: bool = True, encoding: str = 'utf-8') -> None: + + if builder: + warnings.warn('builder argument for generate_autosummary_docs() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + + if template_dir: + warnings.warn('template_dir argument for generate_autosummary_docs() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + + showed_sources = list(sorted(sources)) + if len(showed_sources) > 20: + showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] + logger.info(__('[autosummary] generating autosummary for: %s') % + ', '.join(showed_sources)) + + if output_dir: + logger.info(__('[autosummary] writing to %s') % output_dir) + + if base_path is not None: + sources = [os.path.join(base_path, filename) for filename in sources] + + template = AutosummaryRenderer(app) + + # read + items = find_autosummary_in_files(sources) + + # keep track of new files + new_files = [] + + if app: + filename_map = app.config.autosummary_filename_map + else: + filename_map = {} + + # write + for entry in sorted(set(items), key=str): + if entry.path is None: + # The corresponding autosummary:: directive did not have + # a :toctree: option + continue + + path = output_dir or os.path.abspath(entry.path) + ensuredir(path) + + try: + name, obj, parent, modname = import_by_name(entry.name, grouped_exception=True) + qualname = name.replace(modname + ".", "") + except ImportExceptionGroup as exc: + try: + # try to import as an instance attribute + name, obj, parent, modname = import_ivar_by_name(entry.name) + qualname = name.replace(modname + ".", "") + except ImportError as exc2: + if exc2.__cause__: + exceptions: List[BaseException] = exc.exceptions + [exc2.__cause__] + else: + exceptions = exc.exceptions + [exc2] + + errors = list(set("* %s: %s" % (type(e).__name__, e) for e in exceptions)) + logger.warning(__('[autosummary] failed to import %s.\nPossible hints:\n%s'), + entry.name, '\n'.join(errors)) + continue + + context: Dict[str, Any] = {} + if app: + context.update(app.config.autosummary_context) + + content = generate_autosummary_content(name, obj, parent, template, entry.template, + imported_members, app, entry.recursive, context, + modname, qualname) + try: + py_source_rel = get_full_modname(modname, qualname).replace('.', '/') + '.py' + except: + logger.warning(name) + py_source_rel = '' + + re_view = f"\n.. image:: https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/{app.config.docs_branch}/" + \ + f"resource/_static/logo_source_en.svg\n :target: " + app.config.giturl + \ + f"{app.config.copy_repo}/blob/{app.config.branch}/" + app.config.repo_whl + \ + py_source_rel.split(app.config.cst_module_name)[-1] + '\n :alt: View Source On Gitee\n\n' + + if re_view not in content and py_source_rel: + content = re.sub('([=]{5,})\n', r'\1\n' + re_view, content, 1) + filename = os.path.join(path, filename_map.get(name, name) + suffix) + if os.path.isfile(filename): + with open(filename, encoding=encoding) as f: + old_content = f.read() + + if content == old_content: + continue + elif overwrite: # content has changed + with open(filename, 'w', encoding=encoding) as f: + f.write(content) + new_files.append(filename) + else: + with open(filename, 'w', encoding=encoding) as f: + f.write(content) + new_files.append(filename) + + # descend recursively to new files + if new_files: + generate_autosummary_docs(new_files, output_dir=output_dir, + suffix=suffix, base_path=base_path, + builder=builder, template_dir=template_dir, + imported_members=imported_members, app=app, + overwrite=overwrite) + + +# -- Finding documented entries in files --------------------------------------- + +def find_autosummary_in_files(filenames: List[str]) -> List[AutosummaryEntry]: + """Find out what items are documented in source/*.rst. + + See `find_autosummary_in_lines`. + """ + documented: List[AutosummaryEntry] = [] + for filename in filenames: + with open(filename, encoding='utf-8', errors='ignore') as f: + lines = f.read().splitlines() + documented.extend(find_autosummary_in_lines(lines, filename=filename)) + return documented + + +def find_autosummary_in_docstring(name: str, module: str = None, filename: str = None + ) -> List[AutosummaryEntry]: + """Find out what items are documented in the given object's docstring. + + See `find_autosummary_in_lines`. + """ + if module: + warnings.warn('module argument for find_autosummary_in_docstring() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) + + try: + real_name, obj, parent, modname = import_by_name(name, grouped_exception=True) + lines = pydoc.getdoc(obj).splitlines() + return find_autosummary_in_lines(lines, module=name, filename=filename) + except AttributeError: + pass + except ImportExceptionGroup as exc: + errors = list(set("* %s: %s" % (type(e).__name__, e) for e in exc.exceptions)) + print('Failed to import %s.\nPossible hints:\n%s' % (name, '\n'.join(errors))) + except SystemExit: + print("Failed to import '%s'; the module executes module level " + "statement and it might call sys.exit()." % name) + return [] + + +def find_autosummary_in_lines(lines: List[str], module: str = None, filename: str = None + ) -> List[AutosummaryEntry]: + """Find out what items appear in autosummary:: directives in the + given lines. + + Returns a list of (name, toctree, template) where *name* is a name + of an object and *toctree* the :toctree: path of the corresponding + autosummary directive (relative to the root of the file name), and + *template* the value of the :template: option. *toctree* and + *template* ``None`` if the directive does not have the + corresponding options set. + """ + autosummary_re = re.compile(r'^(\s*)\.\.\s+(ms[a-z]*)?autosummary::\s*') + automodule_re = re.compile( + r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$') + module_re = re.compile( + r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$') + autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?') + recursive_arg_re = re.compile(r'^\s+:recursive:\s*$') + toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') + template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$') + + documented: List[AutosummaryEntry] = [] + + recursive = False + toctree: str = None + template = None + current_module = module + in_autosummary = False + base_indent = "" + + for line in lines: + if in_autosummary: + m = recursive_arg_re.match(line) + if m: + recursive = True + continue + + m = toctree_arg_re.match(line) + if m: + toctree = m.group(1) + if filename: + toctree = os.path.join(os.path.dirname(filename), + toctree) + continue + + m = template_arg_re.match(line) + if m: + template = m.group(1).strip() + continue + + if line.strip().startswith(':'): + continue # skip options + + m = autosummary_item_re.match(line) + if m: + name = m.group(1).strip() + if name.startswith('~'): + name = name[1:] + if current_module and \ + not name.startswith(current_module + '.'): + name = "%s.%s" % (current_module, name) + documented.append(AutosummaryEntry(name, toctree, template, recursive)) + continue + + if not line.strip() or line.startswith(base_indent + " "): + continue + + in_autosummary = False + + m = autosummary_re.match(line) + if m: + in_autosummary = True + base_indent = m.group(1) + recursive = False + toctree = None + template = None + continue + + m = automodule_re.search(line) + if m: + current_module = m.group(1).strip() + # recurse into the automodule docstring + documented.extend(find_autosummary_in_docstring( + current_module, filename=filename)) + continue + + m = module_re.match(line) + if m: + current_module = m.group(2) + continue + + return documented + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + usage='%(prog)s [OPTIONS] ...', + epilog=__('For more information, visit .'), + description=__(""" +Generate ReStructuredText using autosummary directives. + +sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates +the reStructuredText files from the autosummary directives contained in the +given input files. + +The format of the autosummary directive is documented in the +``sphinx.ext.autosummary`` Python module and can be read using:: + + pydoc sphinx.ext.autosummary +""")) + + parser.add_argument('--version', action='version', dest='show_version', + version='%%(prog)s %s' % __display_version__) + + parser.add_argument('source_file', nargs='+', + help=__('source files to generate rST files for')) + + parser.add_argument('-o', '--output-dir', action='store', + dest='output_dir', + help=__('directory to place all output in')) + parser.add_argument('-s', '--suffix', action='store', dest='suffix', + default='rst', + help=__('default suffix for files (default: ' + '%(default)s)')) + parser.add_argument('-t', '--templates', action='store', dest='templates', + default=None, + help=__('custom template directory (default: ' + '%(default)s)')) + parser.add_argument('-i', '--imported-members', action='store_true', + dest='imported_members', default=False, + help=__('document imported members (default: ' + '%(default)s)')) + parser.add_argument('-a', '--respect-module-all', action='store_true', + dest='respect_module_all', default=False, + help=__('document exactly the members in module __all__ attribute. ' + '(default: %(default)s)')) + + return parser + + +def main(argv: List[str] = sys.argv[1:]) -> None: + sphinx.locale.setlocale(locale.LC_ALL, '') + sphinx.locale.init_console(os.path.join(package_dir, 'locale'), 'sphinx') + translator, _ = sphinx.locale.init([], None) + + app = DummyApplication(translator) + logging.setup(app, sys.stdout, sys.stderr) # type: ignore + setup_documenters(app) + args = get_parser().parse_args(argv) + + if args.templates: + app.config.templates_path.append(path.abspath(args.templates)) + app.config.autosummary_ignore_module_all = not args.respect_module_all # type: ignore + + generate_autosummary_docs(args.source_file, args.output_dir, + '.' + args.suffix, + imported_members=args.imported_members, + app=app) + + +if __name__ == '__main__': + main() diff --git a/docs/mindscience/docs/_ext/rename_include.py b/docs/mindscience/docs/_ext/rename_include.py new file mode 100644 index 0000000000000000000000000000000000000000..bf7dea25f3ee7fd371659e80a3551439fbddee5a --- /dev/null +++ b/docs/mindscience/docs/_ext/rename_include.py @@ -0,0 +1,60 @@ +"""Rename .rst file to .txt file for include directive.""" +import os +import re +import glob +import logging + +logging.basicConfig(level=logging.WARNING, format='%(message)s') +logger = logging.getLogger(__name__) + +origin = "rst" +replace = "txt" + +include_re = re.compile(r'\.\. include::\s+(.*?)(\.rst|\.txt)') +include_re_sub = re.compile(rf'(\.\. include::\s+(.*?))\.{origin}') + +# Specified file_name lists excluded from rename procedure. +whitepaper = ['operations.rst'] + +def repl(matchobj): + """Replace functions for matched.""" + if matchobj.group(2).split('/')[-1] + f'.{origin}' in whitepaper: + return matchobj.group(0) + return rf'{matchobj.group(1)}.{replace}' + +def rename_include(api_dir): + """ + Rename .rst file to .txt file for include directive. + + api_dir - api path relative. + """ + tar = [] + for root, _, files in os.walk(api_dir): + for file in files: + if not file.endswith('.rst'): + continue + try: + with open(os.path.join(root, file), 'r+', encoding='utf-8') as f: + content = f.read() + tar_ = include_re.findall(content) + if tar_: + tar_ = [i[0].split('/')[-1]+f'.{origin}' for i in tar_] + tar.extend(tar_) + sub = include_re_sub.findall(content) + if sub: + content_ = include_re_sub.sub(repl, content) + f.seek(0) + f.truncate() + f.write(content_) + except UnicodeDecodeError: + # pylint: disable=logging-fstring-interpolation + logger.warning(f"UnicodeDecodeError for: {file}") + + all_rst = glob.glob(f'{api_dir}/**/*.{origin}', recursive=True) + + for i in all_rst: + if os.path.dirname(i).endswith("api_python") or os.path.basename(i) in whitepaper: + continue + name = os.path.basename(i) + if name in tar: + os.rename(i, i.replace(f'.{origin}', f'.{replace}')) diff --git a/docs/mindscience/docs/source_en/_templates/classtemplate.rst b/docs/mindscience/docs/source_en/_templates/classtemplate.rst index 37a8e95499c8343ad3f8e02d5c9095215fd9010a..0bd7d0a7f6d3ad7098a1b6b4b9b77f25fdd6a051 100644 --- a/docs/mindscience/docs/source_en/_templates/classtemplate.rst +++ b/docs/mindscience/docs/source_en/_templates/classtemplate.rst @@ -3,7 +3,14 @@ .. currentmodule:: {{ module }} -{% if objname in [] %} +{% if fullname=="mindscience.models.transformer.Attention" %} +{{ fullname | underline }} + +.. autoclass:: {{ name }} + :exclude-members: get_qkv, mask_scores, merge_mask, construct + :members: + +{% elif objname in [] %} {{ fullname | underline }} .. autofunction:: {{ fullname }} diff --git a/docs/mindscience/docs/source_en/conf.py b/docs/mindscience/docs/source_en/conf.py index df69e730a643370485b406fea6aa25177a14fa3d..43cfb4672fc24d96f1752483db8e1915b9d68844 100644 --- a/docs/mindscience/docs/source_en/conf.py +++ b/docs/mindscience/docs/source_en/conf.py @@ -12,6 +12,7 @@ # import os import sys +import shutil import IPython import re import sphinx @@ -191,6 +192,73 @@ with open(autodoc_source_path, "r+", encoding="utf8") as f: sys.path.append(os.path.abspath('../../../../resource/sphinx_ext')) import nbsphinx_mod +import mindformers + +moment_dir = os.path.dirname(__file__) +if os.path.exists(os.path.join(moment_dir, 'index.rst')): + os.remove(os.path.join(moment_dir, 'index.rst')) +shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'docs/en/index.rst'), + os.path.join(moment_dir, 'index.rst')) + +copy_path = 'docs/en/api' +src_dir = os.path.join(os.getenv("MSC_PATH"), copy_path) +present_path = os.path.dirname(__file__) +target_dir = './api' + +if os.path.isdir(target_dir): + shutil.rmtree(target_dir) +os.makedirs('./api', exist_ok=True) +for i in os.listdir(src_dir): + if os.path.isfile(os.path.join(src_dir,i)): + shutil.copy(os.path.join(src_dir,i),'./api/'+i) + else: + shutil.copytree(os.path.join(src_dir,i),'./api/'+i) + +# 组件介绍处理 +component_name = ['MindChem', 'MindEarth', 'MindEnergy', 'MindFlow', 'MindSPONGE'] +spec_copy=[] +for name in component_name: + minds_p = name + '/README.md' + docs_p = name + '.md' + spec_copy.append([minds_p, docs_p]) + +moment_dir = os.path.dirname(__file__) +for minds_p, f_p in spec_copy: + ori_p = os.path.join(os.getenv("MSC_PATH"), minds_p) + if os.path.exists(os.path.join(moment_dir, f_p)): + os.remove(os.path.join(moment_dir, f_p)) + shutil.copy(ori_p, os.path.join(moment_dir, f_p)) + +for file_name in component_name: + file_path = os.path.join(moment_dir, f"{file_name}.md") + with open(file_path, 'r+', encoding='utf-8') as f: + content = f.read() + content = re.sub(r'.*?(README_CN.md).*\n.*\n', '', content) + content = re.sub(r'^\s*\[\!\[.*?\]\(.*?\)\]\(.*?\)\s*$', '', content, flags=re.M) + content = re.sub(r'!\[MindSPONGE标志\]\(.*?"MindSPONGE logo"\)', '', content) + content = re.sub(r'## Contents\n[\s\S]*?\n---\n?', '', content) + content = re.sub(r'## Table of Contents\n*\s*(?:- \[.*?\]\(.*?\)\n*\s*)*', '', content) + content = re.sub(r'(?ms)^\s*#{1,6}\s*(Contribution Guide|License)\b.*', '', content) + content = re.sub(r'\n{2,}', '\n\n', content).strip() + f.seek(0) + f.truncate() + f.write(content) + +if not os.path.exists(os.path.join(moment_dir, 'install.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'install.md'), + os.path.join(moment_dir, 'install.md')) + +if not os.path.exists(os.path.join(moment_dir, 'quick_start.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'quick_start.md'), + os.path.join(moment_dir, 'quick_start.md')) + +if not os.path.exists(os.path.join(moment_dir, 'CONTRIBUTING.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'CONTRIBUTION.md'), + os.path.join(moment_dir, 'CONTRIBUTING.md')) + +if not os.path.exists(os.path.join(moment_dir, 'RELEASE.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'RELEASE.md'), + os.path.join(moment_dir, 'RELEASE.md')) sys.path.append(os.path.abspath('../../../../resource/search')) import search_code diff --git a/docs/mindscience/docs/source_zh_cn/conf.py b/docs/mindscience/docs/source_zh_cn/conf.py index 1a5b35b04a305f7032264e7c34cd2946dec6e2c7..f204fe16b3886e4e0fc5c0e200314ebdf4ecda47 100644 --- a/docs/mindscience/docs/source_zh_cn/conf.py +++ b/docs/mindscience/docs/source_zh_cn/conf.py @@ -207,17 +207,91 @@ with open(autodoc_source_path, "r+", encoding="utf8") as f: exec(get_param_func_str, sphinx_autodoc.__dict__) exec(code_str, sphinx_autodoc.__dict__) +moment_dir = os.path.dirname(__file__) +if os.path.exists(os.path.join(moment_dir, 'index.rst')): + os.remove(os.path.join(moment_dir, 'index.rst')) +shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'docs/zh_cn/index.rst'), + os.path.join(moment_dir, 'index.rst')) + +copy_path = 'docs/zh_cn/api' +src_dir = os.path.join(os.getenv("MSC_PATH"), copy_path) + +present_path = os.path.dirname(__file__) +target_dir = './api' + +if os.path.isdir(target_dir): + shutil.rmtree(target_dir) +os.makedirs('./api', exist_ok=True) +for i in os.listdir(src_dir): + if os.path.isfile(os.path.join(src_dir,i)): + shutil.copy(os.path.join(src_dir,i),'./api/'+i) + else: + shutil.copytree(os.path.join(src_dir,i),'./api/'+i) + +# 组件介绍处理 +component_name = ['MindChem', 'MindEarth', 'MindEnergy', 'MindFlow', 'MindSPONGE'] +spec_copy=[] +for name in component_name: + minds_p = name + '/README_CN.md' + docs_p = name + '.md' + spec_copy.append([minds_p, docs_p]) + +moment_dir = os.path.dirname(__file__) +for minds_p, f_p in spec_copy: + ori_p = os.path.join(os.getenv("MSC_PATH"), minds_p) + if os.path.exists(os.path.join(moment_dir, f_p)): + os.remove(os.path.join(moment_dir, f_p)) + shutil.copy(ori_p, os.path.join(moment_dir, f_p)) + +for file_name in component_name: + file_path = os.path.join(moment_dir, f"{file_name}.md") + with open(file_path, 'r+', encoding='utf-8') as f: + content = f.read() + if file_name != 'MindSPONGE': + content = re.sub(r'.*?(README.md).*\n.*\n', '', content, count=1) + content = re.sub(r'^\s*\[\!\[.*?\]\(.*?\)\]\(.*?\)\s*$', '', content, flags=re.M) + content = re.sub(r'!\[MindSPONGE标志\]\(.*?"MindSPONGE logo"\)', '', content) + content = re.sub(r'## 目录\n[\s\S]*?\n---\n?', '', content) + content = re.sub(r'## 目录\n*\s*(?:- \[.*?\]\(.*?\)\n*\s*)*', '', content) + content = re.sub(r'(?ms)^\s*#{1,6}\s*(贡献指南|许可证)\b.*', '', content) + content = re.sub(r'\n{2,}', '\n\n', content).strip() + f.seek(0) + f.truncate() + f.write(content) + +if not os.path.exists(os.path.join(moment_dir, 'install.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'install_CN.md'), + os.path.join(moment_dir, 'install.md')) + +if not os.path.exists(os.path.join(moment_dir, 'quick_start.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'quick_start_CN.md'), + os.path.join(moment_dir, 'quick_start.md')) + +if not os.path.exists(os.path.join(moment_dir, 'CONTRIBUTING.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'CONTRIBUTION_CN.md'), + os.path.join(moment_dir, 'CONTRIBUTING.md')) + +if not os.path.exists(os.path.join(moment_dir, 'RELEASE.md')): + shutil.copy(os.path.join(os.getenv("MSC_PATH"), 'RELEASE_CN.md'), + os.path.join(moment_dir, 'RELEASE.md')) + +# 提取样例、支持平台至中文 +from sphinx import directives +with open('../_ext/overwriteobjectiondirective.txt', 'r', encoding="utf8") as f: + exec(f.read(), directives.__dict__) + sys.path.append(os.path.abspath('../../../../resource/search')) import search_code sys.path.append(os.path.abspath('../../../../resource/custom_directives')) from custom_directives import IncludeCodeDirective -from myautosummary import MsPlatformAutoSummary, MsNoteAutoSummary, MsCnPlatformAutoSummary +from myautosummary import MsPlatformAutoSummary, MsCnAutoSummary, MsNoteAutoSummary, MsCnPlatformAutoSummary rst_files = set([i.replace('.rst', '') for i in glob.glob('./**/*.rst', recursive=True)]) def setup(app): app.add_directive('msplatformautosummary', MsPlatformAutoSummary) + app.add_directive('mscnautosummary', MsCnAutoSummary) app.add_directive('msnoteautosummary', MsNoteAutoSummary) app.add_directive('mscnplatformautosummary', MsCnPlatformAutoSummary) app.add_directive('includecode', IncludeCodeDirective) diff --git a/tools/generate_html/base_version.json b/tools/generate_html/base_version.json index b9314a99597a093e1bed67379b6d9ae0b333b2bf..612880c6ef75e38ef98eae5d742ea0eb815f2451 100644 --- a/tools/generate_html/base_version.json +++ b/tools/generate_html/base_version.json @@ -331,5 +331,14 @@ }, "repo_name": "msadapter", "theme": "theme-docs" + }, + { + "version": "master", + "label": { + "zh": "MindSpore Science", + "en": "MindSpore Science" + }, + "repo_name": "mindscience", + "theme": "theme-docs" } ] \ No newline at end of file diff --git a/tools/generate_html/daily.json b/tools/generate_html/daily.json index c26ee8fa679ca83087d099502940c01e2e79f706..8cbb7913798e7b873d20340c3e6018155d7847e5 100644 --- a/tools/generate_html/daily.json +++ b/tools/generate_html/daily.json @@ -89,5 +89,15 @@ "whl_name" : "mindquantum-.*-cp39-cp39-linux_x86_64.whl$", "environ" : "MQ_PATH", "uninstall_name" : "mindquantum" +}, +{ + "id" : 10 , + "name" : "mindscience", + "branch" : "master", + "whl_search" : "ascend/x86_64/", + "whl_path" : "/mindscience/version/", + "whl_name" : "mindscience-.*-linux_x86_64.whl$", + "environ" : "MSC_PATH", + "uninstall_name" : "mindscience" } -] +] \ No newline at end of file diff --git a/tools/generate_html/run.py b/tools/generate_html/run.py index 6b677721fd0ea229089520910e9b97075870c787..2f97c7f12ea03bda07b691abd8961fe3b69ba472 100644 --- a/tools/generate_html/run.py +++ b/tools/generate_html/run.py @@ -262,8 +262,7 @@ def main(version, user, pd, WGETDIR, release_url, generate_list): subprocess.run(cmd_reppath) # 生成version.json后续放入_static/js/下 - if data[i]['name'] != "mindscience": - generate_version_json(data[i]['name'], data[i]["branch"], data_b, flag_dev, target_version) + generate_version_json(data[i]['name'], data[i]["branch"], data_b, flag_dev, target_version) # 卸载原来已有的安装包, 以防冲突 if data[i]['uninstall_name']: @@ -276,7 +275,7 @@ def main(version, user, pd, WGETDIR, release_url, generate_list): if version == "daily" or flag_dev: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) s = requests.session() - if data[i]['name'] == "reinforcement" or data[i]['name'] == "recommender": + if data[i]['name'] == "reinforcement" or data[i]['name'] == "recommender" or data[i]['name'] == "mindscience": wgetdir = WGETDIR + "mindspore-lab" else: wgetdir = WGETDIR + "mindspore" @@ -446,8 +445,6 @@ def main(version, user, pd, WGETDIR, release_url, generate_list): ArraySource[data[i]['name']] = html_branch elif data[i]['name'] == "mindspore": ArraySource[data[i]['name']] = html_branch - elif data[i]['name'] == "mindscience": - pass else: ArraySource[data[i]['name'] + '/docs'] = html_branch