diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 2a1a2cdd8c..0be13a9fe0 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -132,6 +132,22 @@ subprojects: # checks: # - "test-on-tpus" + - id: "fabric: Docs" + paths: + - "src/lightning/fabric/**" + - "src/lightning_fabric/*" + - "docs/source-fabric/**" + - ".github/workflows/docs-checks.yml" + - "requirements/docs.txt" + - "requirements/fabric/**" + - "setup.py" + - "pyproject.toml" # includes metadata used in the package creation + - "!*.md" + - "!**/*.md" + checks: + - "make-doctest (fabric)" + - "make-html (fabric)" + - id: "pytorch_lightning: Docs" paths: - "src/lightning/pytorch/**" diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 4b7acf4377..ecaf1a7623 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -39,7 +39,7 @@ jobs: strategy: fail-fast: false matrix: - pkg-name: ["app", "pytorch"] + pkg-name: ["app", "fabric", "pytorch"] steps: - uses: actions/checkout@v3 with: @@ -47,14 +47,14 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - uses: aws-actions/configure-aws-credentials@v1 - if: ${{ matrix.pkg-name == 'app' }} + if: ${{ matrix.pkg-name != 'pytorch' }} with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_ID }} aws-region: us-east-1 - run: aws s3 sync s3://sphinx-packages/ pypi/ - if: ${{ matrix.pkg-name == 'app' }} + if: ${{ matrix.pkg-name != 'pytorch' }} # Note: This uses an internal pip API and may not always work # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow @@ -85,7 +85,7 @@ jobs: strategy: fail-fast: false matrix: - pkg-name: ["app", "pytorch"] + pkg-name: ["app", "fabric", "pytorch"] steps: - uses: actions/checkout@v3 with: diff --git a/docs/source-app/conf.py b/docs/source-app/conf.py index f507160f8d..0ef7cc9141 100644 --- a/docs/source-app/conf.py +++ b/docs/source-app/conf.py @@ -134,7 +134,7 @@ master_doc = "index" # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source-app directory, that match files and # directories to ignore when looking for source-app files. @@ -396,5 +396,13 @@ doctest_global_setup = """ import importlib import os import lightning as L + +from lightning.fabric.loggers.tensorboard import _TENSORBOARD_AVAILABLE, _TENSORBOARDX_AVAILABLE """ coverage_skip_undoc_in_source = True + +# skip false positive linkcheck errors from anchors +linkcheck_anchors = False + +# ignore all links in any CHANGELOG file +linkcheck_exclude_documents = [r"^(.*\/)*CHANGELOG.*$"] diff --git a/docs/source-fabric/Makefile b/docs/source-fabric/Makefile new file mode 100644 index 0000000000..268e09561b --- /dev/null +++ b/docs/source-fabric/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = -T -W +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = ../build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source-fabric/_static/copybutton.js b/docs/source-fabric/_static/copybutton.js new file mode 100644 index 0000000000..453363ce9e --- /dev/null +++ b/docs/source-fabric/_static/copybutton.js @@ -0,0 +1,64 @@ +/* Copied from the official Python docs: https://docs.python.org/3/_static/copybutton.js */ +$(document).ready(function() { + /* Add a [>>>] button on the top-right corner of code samples to hide + * the >>> and ... prompts and the output and thus make the code + * copyable. */ + var div = $('.highlight-python .highlight,' + + '.highlight-python3 .highlight,' + + '.highlight-pycon .highlight,' + + '.highlight-default .highlight'); + var pre = div.find('pre'); + + // get the styles from the current theme + pre.parent().parent().css('position', 'relative'); + var hide_text = 'Hide the prompts and output'; + var show_text = 'Show the prompts and output'; + var border_width = pre.css('border-top-width'); + var border_style = pre.css('border-top-style'); + var border_color = pre.css('border-top-color'); + var button_styles = { + 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', + 'border-color': border_color, 'border-style': border_style, + 'border-width': border_width, 'color': border_color, 'text-size': '75%', + 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', + 'border-radius': '0 3px 0 0' + } + + // create and add the button to all the code blocks that contain >>> + div.each(function(index) { + var jthis = $(this); + if (jthis.find('.gp').length > 0) { + var button = $('>>>'); + button.css(button_styles) + button.attr('title', hide_text); + button.data('hidden', 'false'); + jthis.prepend(button); + } + // tracebacks (.gt) contain bare text elements that need to be + // wrapped in a span to work with .nextUntil() (see later) + jthis.find('pre:has(.gt)').contents().filter(function() { + return ((this.nodeType == 3) && (this.data.trim().length > 0)); + }).wrap(''); + }); + + // define the behavior of the button when it's clicked + $('.copybutton').click(function(e){ + e.preventDefault(); + var button = $(this); + if (button.data('hidden') === 'false') { + // hide the code output + button.parent().find('.go, .gp, .gt').hide(); + button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); + button.css('text-decoration', 'line-through'); + button.attr('title', show_text); + button.data('hidden', 'true'); + } else { + // show the code output + button.parent().find('.go, .gp, .gt').show(); + button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); + button.css('text-decoration', 'none'); + button.attr('title', hide_text); + button.data('hidden', 'false'); + } + }); +}); diff --git a/docs/source-fabric/_static/images/icon.svg b/docs/source-fabric/_static/images/icon.svg new file mode 100644 index 0000000000..e88fc19036 --- /dev/null +++ b/docs/source-fabric/_static/images/icon.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/docs/source-fabric/_static/images/logo-large.svg b/docs/source-fabric/_static/images/logo-large.svg new file mode 100644 index 0000000000..39531f95e9 --- /dev/null +++ b/docs/source-fabric/_static/images/logo-large.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/docs/source-fabric/_static/images/logo-small.svg b/docs/source-fabric/_static/images/logo-small.svg new file mode 100644 index 0000000000..1f523a57c4 --- /dev/null +++ b/docs/source-fabric/_static/images/logo-small.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/docs/source-fabric/_static/images/logo.png b/docs/source-fabric/_static/images/logo.png new file mode 100644 index 0000000000..392c965b24 Binary files /dev/null and b/docs/source-fabric/_static/images/logo.png differ diff --git a/docs/source-fabric/_static/images/logo.svg b/docs/source-fabric/_static/images/logo.svg new file mode 100644 index 0000000000..60efaa29b8 --- /dev/null +++ b/docs/source-fabric/_static/images/logo.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/docs/source-fabric/_static/main.css b/docs/source-fabric/_static/main.css new file mode 100644 index 0000000000..82aa8b338a --- /dev/null +++ b/docs/source-fabric/_static/main.css @@ -0,0 +1,3 @@ +col { + width: 50% !important; +} diff --git a/docs/source-fabric/_templates/autosummary/module.rst b/docs/source-fabric/_templates/autosummary/module.rst new file mode 100644 index 0000000000..704ac61bf7 --- /dev/null +++ b/docs/source-fabric/_templates/autosummary/module.rst @@ -0,0 +1,41 @@ +{{ name | escape | underline }} + +.. currentmodule:: {{ fullname }} + +{% block functions %} +{% if functions %} +.. rubric:: Functions + +.. autosummary:: + :nosignatures: +{% for item in functions %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} + +{% block classes %} +{% if classes %} +.. rubric:: Classes + +.. autosummary:: + :nosignatures: +{% for item in classes %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} + +{% block exceptions %} +{% if exceptions %} +.. rubric:: Exceptions + +.. autosummary:: + :nosignatures: +{% for item in exceptions %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} + +.. automodule:: {{ fullname }} diff --git a/docs/source-fabric/_templates/classtemplate.rst b/docs/source-fabric/_templates/classtemplate.rst new file mode 100644 index 0000000000..482db1324d --- /dev/null +++ b/docs/source-fabric/_templates/classtemplate.rst @@ -0,0 +1,14 @@ +.. role:: hidden + :class: hidden-section +.. currentmodule:: {{ module }} + + +{{ name | underline }} + +.. autoclass:: {{ name }} + :members: + + +.. + autogenerated from source-fabric/_templates/classtemplate.rst + note it does not have :inherited-members: diff --git a/docs/source-fabric/_templates/classtemplate_no_index.rst b/docs/source-fabric/_templates/classtemplate_no_index.rst new file mode 100644 index 0000000000..858c37b515 --- /dev/null +++ b/docs/source-fabric/_templates/classtemplate_no_index.rst @@ -0,0 +1,12 @@ +:orphan: + +.. role:: hidden + :class: hidden-section +.. currentmodule:: {{ module }} + + +{{ name | underline }} + +.. autoclass:: {{ name }} + :members: + :noindex: diff --git a/docs/source-fabric/_templates/layout.html b/docs/source-fabric/_templates/layout.html new file mode 100644 index 0000000000..48de111416 --- /dev/null +++ b/docs/source-fabric/_templates/layout.html @@ -0,0 +1,10 @@ +{% extends "!layout.html" %} + + +{% block footer %} +{{ super() }} + + +{% endblock %} diff --git a/docs/source-fabric/_templates/theme_variables.jinja b/docs/source-fabric/_templates/theme_variables.jinja new file mode 100644 index 0000000000..447390ecc9 --- /dev/null +++ b/docs/source-fabric/_templates/theme_variables.jinja @@ -0,0 +1,18 @@ +{%- set external_urls = { + 'github': 'https://github.com/Lightning-AI/lightning', + 'github_issues': 'https://github.com/Lightning-AI/lightning/issues', + 'contributing': 'https://github.com/Lightning-AI/lightning/blob/master/.github/CONTRIBUTING.md', + 'governance': 'https://github.com/Lightning-AI/lightning/blob/master/docs/source-pytorch/governance.rst', + 'docs': 'https://lightning.rtfd.io/en/latest', + 'twitter': 'https://twitter.com/PyTorchLightnin', + 'discuss': 'https://pytorch-lightning.slack.com', + 'tutorials': 'https://pt-lightning.readthedocs.io/en/latest/#tutorials', + 'previous_pytorch_versions': 'https://pt-lightning.rtfd.io/en/latest/', + 'home': 'https://lightning.ai/', + 'get_started': 'https://pt-lightning.readthedocs.io/en/latest/introduction_guide.html', + 'features': 'https://pt-lightning.rtfd.io/en/latest/', + 'blog': 'https://www.pytorchlightning.ai/blog', + 'resources': 'https://pt-lightning.readthedocs.io/en/latest/#community-examples', + 'support': 'https://pt-lightning.rtfd.io/en/latest/', +} +-%} diff --git a/docs/source-pytorch/fabric/advanced/distributed_communication.rst b/docs/source-fabric/advanced/distributed_communication.rst similarity index 100% rename from docs/source-pytorch/fabric/advanced/distributed_communication.rst rename to docs/source-fabric/advanced/distributed_communication.rst diff --git a/docs/source-pytorch/fabric/advanced/gradient_accumulation.rst b/docs/source-fabric/advanced/gradient_accumulation.rst similarity index 100% rename from docs/source-pytorch/fabric/advanced/gradient_accumulation.rst rename to docs/source-fabric/advanced/gradient_accumulation.rst diff --git a/docs/source-pytorch/fabric/api/fabric_args.rst b/docs/source-fabric/api/fabric_args.rst similarity index 98% rename from docs/source-pytorch/fabric/api/fabric_args.rst rename to docs/source-fabric/api/fabric_args.rst index 3ee1fe9e95..eb7a7916c1 100644 --- a/docs/source-pytorch/fabric/api/fabric_args.rst +++ b/docs/source-fabric/api/fabric_args.rst @@ -140,7 +140,7 @@ See also: :doc:`../fundamentals/precision` plugins ======= -:ref:`Plugins` allow you to connect arbitrary backends, precision libraries, clusters, etc. For example: +Plugins allow you to connect arbitrary backends, precision libraries, clusters, etc. For example: To define your own behavior, subclass the relevant class and pass it in. Here's an example linking up your own :class:`~lightning.fabric.plugins.environments.ClusterEnvironment`. diff --git a/docs/source-pytorch/fabric/api/fabric_methods.rst b/docs/source-fabric/api/fabric_methods.rst similarity index 100% rename from docs/source-pytorch/fabric/api/fabric_methods.rst rename to docs/source-fabric/api/fabric_methods.rst diff --git a/docs/source-pytorch/fabric/api/utilities.rst b/docs/source-fabric/api/utilities.rst similarity index 100% rename from docs/source-pytorch/fabric/api/utilities.rst rename to docs/source-fabric/api/utilities.rst diff --git a/docs/source-pytorch/fabric/api/api_reference.rst b/docs/source-fabric/api_reference.rst similarity index 89% rename from docs/source-pytorch/fabric/api/api_reference.rst rename to docs/source-fabric/api_reference.rst index 59cd6441c6..13cc0207d1 100644 --- a/docs/source-pytorch/fabric/api/api_reference.rst +++ b/docs/source-fabric/api_reference.rst @@ -1,6 +1,6 @@ :orphan: -.. include:: ../../links.rst +.. include:: links.rst ############# API Reference @@ -13,7 +13,7 @@ Fabric .. currentmodule:: lightning.fabric.fabric .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst @@ -26,7 +26,7 @@ Accelerators .. currentmodule:: lightning.fabric.accelerators .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst @@ -43,7 +43,7 @@ Loggers .. currentmodule:: lightning.fabric.loggers .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst @@ -51,10 +51,6 @@ Loggers CSVLogger TensorBoardLogger - -Plugins -^^^^^^^ - Precision """"""""" @@ -63,7 +59,7 @@ Precision .. currentmodule:: lightning.fabric.plugins.precision .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst @@ -81,7 +77,7 @@ Environments .. currentmodule:: lightning.fabric.plugins.environments .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate_noindex.rst @@ -101,7 +97,7 @@ IO .. currentmodule:: lightning.fabric.plugins.io .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst @@ -116,7 +112,7 @@ Collectives .. currentmodule:: lightning.fabric.plugins.collectives .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst @@ -133,7 +129,7 @@ Strategies .. currentmodule:: lightning.fabric.strategies .. autosummary:: - :toctree: ../../api + :toctree: api :nosignatures: :template: classtemplate.rst diff --git a/docs/source-fabric/conf.py b/docs/source-fabric/conf.py new file mode 100644 index 0000000000..96c8c34346 --- /dev/null +++ b/docs/source-fabric/conf.py @@ -0,0 +1,392 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +import glob +import inspect +import os +import shutil +import sys + +import lai_sphinx_theme + +import lightning + +_PATH_HERE = os.path.abspath(os.path.dirname(__file__)) +_PATH_ROOT = os.path.realpath(os.path.join(_PATH_HERE, "..", "..")) +sys.path.insert(0, os.path.abspath(_PATH_ROOT)) + +SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) + +# -- Project information ----------------------------------------------------- + +# this name shall match the project name in Github as it is used for linking to code +project = "lightning" +copyright = lightning.__copyright__ +author = lightning.__author__ + +# The short X.Y version +version = lightning.__version__ +# The full version, including alpha/beta/rc tags +release = lightning.__version__ + +# Options for the linkcode extension +# ---------------------------------- +github_user = "Lightning-AI" +github_repo = project + +# -- Project documents ------------------------------------------------------- + + +# def _transform_changelog(path_in: str, path_out: str) -> None: +# with open(path_in) as fp: +# chlog_lines = fp.readlines() +# # enrich short subsub-titles to be unique +# chlog_ver = "" +# for i, ln in enumerate(chlog_lines): +# if ln.startswith("## "): +# chlog_ver = ln[2:].split("-")[0].strip() +# elif ln.startswith("### "): +# ln = ln.replace("###", f"### {chlog_ver} -") +# chlog_lines[i] = ln +# with open(path_out, "w") as fp: +# fp.writelines(chlog_lines) + + +# export the READme +# _convert_markdown(os.path.join(_PATH_ROOT, "README.md"), "readme.md") + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. + +needs_sphinx = "4.5" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + # 'sphinxcontrib.mockautodoc', # raises error: directive 'automodule' is already registered ... + # 'sphinxcontrib.fulltoc', # breaks pytorch-theme with unexpected kw argument 'titles_only' + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx_toolbox.collapse", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.imgmath", + "sphinx.ext.autosectionlabel", + "myst_parser", + "sphinx_autodoc_typehints", + "sphinx_copybutton", + "sphinx_paramlinks", + "sphinx_togglebutton", + "lai_sphinx_theme.extensions.lightning", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# myst-parser, forcing to parse all html pages with mathjax +# https://github.com/executablebooks/MyST-Parser/issues/394 +myst_update_mathjax = False +# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html?highlight=anchor#auto-generated-header-anchors +myst_heading_anchors = 3 + +# https://berkeley-stat159-f17.github.io/stat159-f17/lectures/14-sphinx..html#conf.py-(cont.) +# https://stackoverflow.com/questions/38526888/embed-ipython-notebook-in-sphinx-document +# I execute the notebooks manually in advance. If notebooks test the code, +# they should be run at build time. +nbsphinx_execute = "never" +nbsphinx_allow_errors = True +nbsphinx_requirejs_path = "" + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +# source_suffix = ['.rst', '.md', '.ipynb'] +source_suffix = { + ".rst": "restructuredtext", + ".txt": "markdown", + ".md": "markdown", + ".ipynb": "nbsphinx", +} + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + "PULL_REQUEST_TEMPLATE.md", + "**/README.md/*", + "readme.md", + "_templates", + "code_samples/convert_pl_to_app/requirements.txt", + "**/_static/*", +] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "lai_sphinx_theme" +html_theme_path = [os.environ.get("LIT_SPHINX_PATH", lai_sphinx_theme.get_html_theme_path())] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. + +html_theme_options = { + "pytorch_project": lightning.__homepage__, + "analytics_id": "G-D3Q2ESCTZR", + "canonical_url": lightning.__homepage__, + "collapse_navigation": False, + "display_version": True, + "logo_only": False, +} + +html_favicon = "_static/images/icon.svg" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_templates", "_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = project + "-doc" + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + # Latex figure (float) alignment + "figure_align": "htbp", +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, project + ".tex", project + " Documentation", author, "manual"), +] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, project, project + " Documentation", [author], 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + project, + project + " Documentation", + author, + project, + lightning.__docs__, + "Miscellaneous", + ), +] + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ["search.html"] + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "torch": ("https://pytorch.org/docs/stable/", None), + "pytorch_lightning": ("https://pytorch-lightning.readthedocs.io/en/stable/", None), +} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +def setup(app): + # this is for hiding doctest decoration, + # see: http://z4r.github.io/python/2011/12/02/hides-the-prompts-and-output/ + app.add_js_file("copybutton.js") + app.add_css_file("main.css") + + +# Ignoring Third-party packages +# https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule +def _package_list_from_file(file): + list_pkgs = [] + with open(file) as fp: + lines = fp.readlines() + for ln in lines: + found = [ln.index(ch) for ch in list(",=<>#") if ch in ln] + pkg = ln[: min(found)] if found else ln + if pkg.rstrip(): + list_pkgs.append(pkg.rstrip()) + return list_pkgs + + +# define mapping from PyPI names to python imports +PACKAGE_MAPPING = { + "PyYAML": "yaml", +} +MOCK_PACKAGES = [] +if SPHINX_MOCK_REQUIREMENTS: + # mock also base packages when we are on RTD since we don't install them there + MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_ROOT, "requirements.txt")) +MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES] + +autodoc_mock_imports = MOCK_PACKAGES + + +# Resolve function +# This function is used to populate the (source) links in the API +def linkcode_resolve(domain, info): + def find_source(): + # try to find the file and line number, based on code from numpy: + # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286 + obj = sys.modules[info["module"]] + for part in info["fullname"].split("."): + obj = getattr(obj, part) + fname = inspect.getsourcefile(obj) + # https://github.com/rtfd/readthedocs.org/issues/5735 + if any(s in fname for s in ("readthedocs", "rtfd", "checkouts")): + # /home/docs/checkouts/readthedocs.org/user_builds/pytorch_lightning/checkouts/ + # devel/pytorch_lightning/utilities/cls_experiment.py#L26-L176 + path_top = os.path.abspath(os.path.join("..", "..", "..")) + fname = os.path.relpath(fname, start=path_top) + else: + # Local build, imitate master + fname = "master/" + os.path.relpath(fname, start=os.path.abspath("..")) + source, lineno = inspect.getsourcelines(obj) + return fname, lineno, lineno + len(source) - 1 + + if domain != "py" or not info["module"]: + return None + try: + filename = "%s#L%d-L%d" % find_source() + except Exception: + filename = info["module"].replace(".", "/") + ".py" + # import subprocess + # tag = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, + # universal_newlines=True).communicate()[0][:-1] + branch = filename.split("/")[0] + # do mapping from latest tags to master + branch = {"latest": "master", "stable": "master"}.get(branch, branch) + filename = "/".join([branch] + filename.split("/")[1:]) + return f"https://github.com/{github_user}/{github_repo}/blob/{filename}" + + +autosummary_generate = True + +autodoc_member_order = "groupwise" +autoclass_content = "both" +# the options are fixed and will be soon in release, +# see https://github.com/sphinx-doc/sphinx/issues/5459 +autodoc_default_options = { + "members": None, + "methods": None, + # 'attributes': None, + "special-members": "__call__", + "exclude-members": "_abc_impl", + "show-inheritance": True, + "private-members": True, + "noindex": True, +} + +# Sphinx will add “permalinks” for each heading and description environment as paragraph signs that +# become visible when the mouse hovers over them. +# This value determines the text for the permalink; it defaults to "¶". Set it to None or the empty +# string to disable permalinks. +# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_permalinks +# html_add_permalinks = "¶" +# True to prefix each section label with the name of the document it is in, followed by a colon. +# For example, index:Introduction for a section called Introduction that appears in document index.rst. +# Useful for avoiding ambiguity when the same section heading appears in different documents. +# http://www.sphinx-doc.org/en/master/usage/extensions/autosectionlabel.html +autosectionlabel_prefix_document = True + +# only run doctests marked with a ".. doctest::" directive +doctest_test_doctest_blocks = "" +doctest_global_setup = """ +import importlib +import os +import lightning as L + +from lightning_utilities.core.imports import package_available +from lightning import LightningModule, Trainer +from lightning.fabric.loggers.tensorboard import _TENSORBOARD_AVAILABLE, _TENSORBOARDX_AVAILABLE + +_TORCHVISION_AVAILABLE = package_available("torchvision") +""" +coverage_skip_undoc_in_source = True + +# skip false positive linkcheck errors from anchors +linkcheck_anchors = False + +# ignore all links in any CHANGELOG file +linkcheck_exclude_documents = [r"^(.*\/)*CHANGELOG.*$"] diff --git a/docs/source-pytorch/fabric/fundamentals/accelerators.rst b/docs/source-fabric/fundamentals/accelerators.rst similarity index 100% rename from docs/source-pytorch/fabric/fundamentals/accelerators.rst rename to docs/source-fabric/fundamentals/accelerators.rst diff --git a/docs/source-pytorch/fabric/fundamentals/code_structure.rst b/docs/source-fabric/fundamentals/code_structure.rst similarity index 100% rename from docs/source-pytorch/fabric/fundamentals/code_structure.rst rename to docs/source-fabric/fundamentals/code_structure.rst diff --git a/docs/source-pytorch/fabric/fundamentals/convert.rst b/docs/source-fabric/fundamentals/convert.rst similarity index 98% rename from docs/source-pytorch/fabric/fundamentals/convert.rst rename to docs/source-fabric/fundamentals/convert.rst index 7e14df4ad3..55cbd82263 100644 --- a/docs/source-pytorch/fabric/fundamentals/convert.rst +++ b/docs/source-fabric/fundamentals/convert.rst @@ -20,7 +20,7 @@ Here are five easy steps to let :class:`~lightning.fabric.fabric.Fabric` scale y fabric.launch() -**Step 3:** Call :meth:`~lightning_fabric.fabric.Fabric.setup` on each model and optimizer pair and :meth:`~lightning_fabric.fabric.Fabric.setup_dataloaders` on all your data loaders. +**Step 3:** Call :meth:`~lightning.fabric.fabric.Fabric.setup` on each model and optimizer pair and :meth:`~lightning_fabric.fabric.Fabric.setup_dataloaders` on all your data loaders. .. code-block:: python diff --git a/docs/source-pytorch/fabric/fundamentals/launch.rst b/docs/source-fabric/fundamentals/launch.rst similarity index 100% rename from docs/source-pytorch/fabric/fundamentals/launch.rst rename to docs/source-fabric/fundamentals/launch.rst diff --git a/docs/source-pytorch/fabric/fundamentals/notebooks.rst b/docs/source-fabric/fundamentals/notebooks.rst similarity index 100% rename from docs/source-pytorch/fabric/fundamentals/notebooks.rst rename to docs/source-fabric/fundamentals/notebooks.rst diff --git a/docs/source-pytorch/fabric/fundamentals/precision.rst b/docs/source-fabric/fundamentals/precision.rst similarity index 100% rename from docs/source-pytorch/fabric/fundamentals/precision.rst rename to docs/source-fabric/fundamentals/precision.rst diff --git a/docs/source-pytorch/fabric/guide/callbacks.rst b/docs/source-fabric/guide/callbacks.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/callbacks.rst rename to docs/source-fabric/guide/callbacks.rst diff --git a/docs/source-pytorch/fabric/guide/checkpoint.rst b/docs/source-fabric/guide/checkpoint.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/checkpoint.rst rename to docs/source-fabric/guide/checkpoint.rst diff --git a/docs/source-pytorch/fabric/guide/lightning_module.rst b/docs/source-fabric/guide/lightning_module.rst similarity index 92% rename from docs/source-pytorch/fabric/guide/lightning_module.rst rename to docs/source-fabric/guide/lightning_module.rst index 2b2f659b7e..fec49b1427 100644 --- a/docs/source-pytorch/fabric/guide/lightning_module.rst +++ b/docs/source-fabric/guide/lightning_module.rst @@ -7,7 +7,7 @@ Organize Your Code Any raw PyTorch can be converted to Fabric with zero refactoring required, giving maximum flexibility in how you want to organize your projects. However, when developing a project in a team or sharing the code publicly, it can be beneficial to conform to a standard format of how core pieces of the code are organized. -This is what the :doc:`LightningModule <../../common/lightning_module>` was made for! +This is what the :class:`pytorch_lightning.core.module.LightningModule` was made for! Here is how you can neatly separate the research code (model, loss, optimization, etc.) from the "trainer" code (training loop, checkpointing, logging, etc.). @@ -62,7 +62,7 @@ Take these main ingredients and put them in a LightningModule: ... -This is a minimal LightningModule, but there are :doc:`many other useful hooks <../../common/lightning_module>` you can use. +This is a minimal :class:`pytorch_lightning.LightningModule`, but there are `many other useful hooks `_ you can use. ---- diff --git a/docs/source-pytorch/fabric/guide/logging.rst b/docs/source-fabric/guide/logging.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/logging.rst rename to docs/source-fabric/guide/logging.rst diff --git a/docs/source-pytorch/fabric/guide/multi_node/barebones.rst b/docs/source-fabric/guide/multi_node/barebones.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/multi_node/barebones.rst rename to docs/source-fabric/guide/multi_node/barebones.rst diff --git a/docs/source-pytorch/fabric/guide/multi_node/cloud.rst b/docs/source-fabric/guide/multi_node/cloud.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/multi_node/cloud.rst rename to docs/source-fabric/guide/multi_node/cloud.rst diff --git a/docs/source-pytorch/fabric/guide/multi_node/other.rst b/docs/source-fabric/guide/multi_node/other.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/multi_node/other.rst rename to docs/source-fabric/guide/multi_node/other.rst diff --git a/docs/source-pytorch/fabric/guide/multi_node/slurm.rst b/docs/source-fabric/guide/multi_node/slurm.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/multi_node/slurm.rst rename to docs/source-fabric/guide/multi_node/slurm.rst diff --git a/docs/source-pytorch/fabric/guide/trainer_template.rst b/docs/source-fabric/guide/trainer_template.rst similarity index 100% rename from docs/source-pytorch/fabric/guide/trainer_template.rst rename to docs/source-fabric/guide/trainer_template.rst diff --git a/docs/source-pytorch/fabric/fabric.rst b/docs/source-fabric/index.rst similarity index 96% rename from docs/source-pytorch/fabric/fabric.rst rename to docs/source-fabric/index.rst index f19a1628c8..40cd97c0bf 100644 --- a/docs/source-pytorch/fabric/fabric.rst +++ b/docs/source-fabric/index.rst @@ -60,20 +60,20 @@ Fabric is the fast and lightweight way to scale PyTorch models without boilerpla Why Fabric? *********** -Fabric differentiates itself from a fully-fledged trainer like :doc:`Lightning Trainer <../common/trainer>` in these key aspects: +Fabric differentiates itself from a fully-fledged trainer like Lightning :class:`pytorch_lightning.Trainer` in these key aspects: **Fast to implement** There is no need to restructure your code: Just change a few lines in the PyTorch script and you'll be able to leverage Fabric features. **Maximum Flexibility** Write your own training and/or inference logic down to the individual optimizer calls. -You aren't forced to conform to a standardized epoch-based training loop like the one in :doc:`Lightning Trainer <../common/trainer>`. +You aren't forced to conform to a standardized epoch-based training loop like the one in Lightning :class:`pytorch_lightning.Trainer`. You can do flexible iteration based training, meta-learning, cross-validation and other types of optimization algorithms without digging into framework internals. This also makes it super easy to adopt Fabric in existing PyTorch projects to speed-up and scale your models without the compromise on large refactors. Just remember: With great power comes a great responsibility. **Maximum Control** -The :doc:`Lightning Trainer <../common/trainer>` has many built in features to make research simpler with less boilerplate, but debugging it requires some familiarity with the framework internals. +The Lightning :class:`pytorch_lightning.Trainer` has many built-in features to make research simpler with less boilerplate, but debugging it requires some familiarity with the framework internals. In Fabric, everything is opt-in. Think of it as a toolbox: You take out the tools (Fabric functions) you need and leave the other ones behind. This makes it easier to develop and debug your PyTorch code as you gradually add more features to it. Fabric provides important tools to remove undesired boilerplate code (distributed, hardware, checkpoints, logging, ...), but leaves the design and orchestration fully up to you. @@ -344,7 +344,7 @@ API .. displayitem:: :header: Full API Reference :description: Reference of all public classes, methods and functions. Useful for developers. - :button_link: api/api_reference.html + :button_link: api_reference.html :col_css: col-md-4 :height: 150 :tag: intermediate diff --git a/docs/source-fabric/links.rst b/docs/source-fabric/links.rst new file mode 100644 index 0000000000..64ec918bf8 --- /dev/null +++ b/docs/source-fabric/links.rst @@ -0,0 +1,2 @@ +.. _PyTorchJob: https://www.kubeflow.org/docs/components/training/pytorch/ +.. _Kubeflow: https://www.kubeflow.org diff --git a/docs/source-fabric/make.bat b/docs/source-fabric/make.bat new file mode 100644 index 0000000000..9b565142ae --- /dev/null +++ b/docs/source-fabric/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=../build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/source-pytorch/_templates/classtemplate.rst b/docs/source-pytorch/_templates/classtemplate.rst index 398a0ec07c..dc11b74d14 100644 --- a/docs/source-pytorch/_templates/classtemplate.rst +++ b/docs/source-pytorch/_templates/classtemplate.rst @@ -10,5 +10,5 @@ .. - autogenerated from source/_templates/classtemplate.rst + autogenerated from source-pytorch/_templates/classtemplate.rst note it does not have :inherited-members: diff --git a/docs/source-pytorch/conf.py b/docs/source-pytorch/conf.py index d1909c005f..054043745d 100644 --- a/docs/source-pytorch/conf.py +++ b/docs/source-pytorch/conf.py @@ -74,7 +74,7 @@ for md in glob.glob(os.path.join(PATH_ROOT, ".github", "*.md")): shutil.copy(md, os.path.join(PATH_HERE, FOLDER_GENERATED, os.path.basename(md))) # copy also the changelog _transform_changelog( - os.path.join(PATH_ROOT, "src", "lightning", "pytorch", "CHANGELOG.md"), + os.path.join(PATH_ROOT, "src", "lightning", "fabric", "CHANGELOG.md"), os.path.join(PATH_HERE, FOLDER_GENERATED, "CHANGELOG.md"), ) @@ -162,7 +162,7 @@ master_doc = "index" # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -396,9 +396,7 @@ from torch.utils.data import IterableDataset, DataLoader, Dataset from lightning.pytorch import LightningDataModule, LightningModule, Trainer, seed_everything from lightning.pytorch.callbacks import Callback from lightning.pytorch.cli import _JSONARGPARSE_SIGNATURES_AVAILABLE as _JSONARGPARSE_AVAILABLE -from lightning.pytorch.utilities import ( - _TORCHVISION_AVAILABLE, -) +from lightning.pytorch.utilities import _TORCHVISION_AVAILABLE from lightning.fabric.loggers.tensorboard import _TENSORBOARD_AVAILABLE, _TENSORBOARDX_AVAILABLE from lightning.pytorch.loggers.neptune import _NEPTUNE_AVAILABLE from lightning.pytorch.loggers.comet import _COMET_AVAILABLE diff --git a/docs/source-pytorch/index.rst b/docs/source-pytorch/index.rst index 3130ee5ca4..c00dec356a 100644 --- a/docs/source-pytorch/index.rst +++ b/docs/source-pytorch/index.rst @@ -174,7 +174,6 @@ Current Lightning Users common/lightning_module common/trainer - fabric/fabric .. toctree:: :maxdepth: 2 @@ -232,7 +231,6 @@ Current Lightning Users Inference IPU Lightning CLI - Lightning Fabric LightningDataModule LightningModule Lightning Transformers diff --git a/docs/source-pytorch/levels/advanced_level_17.rst b/docs/source-pytorch/levels/advanced_level_17.rst index 4b6b5f0337..f18b5ad101 100644 --- a/docs/source-pytorch/levels/advanced_level_17.rst +++ b/docs/source-pytorch/levels/advanced_level_17.rst @@ -23,14 +23,6 @@ Learn all the ways of owning your raw PyTorch loops with Lighting. :height: 150 :tag: advanced -.. displayitem:: - :header: Use a Raw PyTorch Loop - :description: Migrate complex PyTorch projects to Lightning and push bleeding-edge research with the raw PyTorch loop. - :col_css: col-md-4 - :button_link: ../fabric/fabric.html - :height: 150 - :tag: advanced - .. raw:: html diff --git a/docs/source-pytorch/model/own_your_loop.rst b/docs/source-pytorch/model/own_your_loop.rst index 54839b0120..ad0c88912c 100644 --- a/docs/source-pytorch/model/own_your_loop.rst +++ b/docs/source-pytorch/model/own_your_loop.rst @@ -19,14 +19,6 @@ Use a pure PyTorch training loop :height: 150 :tag: advanced -.. displayitem:: - :header: Use a Raw PyTorch Loop - :description: Migrate complex PyTorch projects to Lightning and push bleeding-edge research with the raw PyTorch loop. - :col_css: col-md-4 - :button_link: ../fabric/fabric.html - :height: 150 - :tag: advanced - .. raw:: html diff --git a/docs/source-pytorch/starter/introduction.rst b/docs/source-pytorch/starter/introduction.rst index aad3b907a3..c126e30f35 100644 --- a/docs/source-pytorch/starter/introduction.rst +++ b/docs/source-pytorch/starter/introduction.rst @@ -325,15 +325,6 @@ For certain types of work at the bleeding-edge of research, Lightning offers exp :image_height: 220px :height: 320 -.. displayitem:: - :header: Lightning Fabric - :description: Full control over loop for migrating complex PyTorch projects. - :col_css: col-md-4 - :image_center: https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/lite.png - :button_link: ../fabric/fabric.html - :image_height: 220px - :height: 320 - .. raw:: html @@ -371,14 +362,6 @@ Depending on your use case, you might want to check one of these out next. :height: 180 :tag: basic -.. displayitem:: - :header: I need my raw PyTorch Loop - :description: Expert-level control for researchers working on the bleeding-edge - :col_css: col-md-3 - :button_link: ../fabric/fabric.html - :height: 180 - :tag: expert - .. displayitem:: :header: Deploy your model :description: Learn how to predict or put your model into production diff --git a/requirements/app/docs.txt b/requirements/app/docs.txt index a677d283ec..e931fcfe66 100644 --- a/requirements/app/docs.txt +++ b/requirements/app/docs.txt @@ -1,6 +1,3 @@ -r ../docs.txt -ipython[notebook] -ipython_genutils - lai-sphinx-theme diff --git a/requirements/fabric/docs.txt b/requirements/fabric/docs.txt new file mode 100644 index 0000000000..506eb31a3c --- /dev/null +++ b/requirements/fabric/docs.txt @@ -0,0 +1,4 @@ +-r ../docs.txt + +lai-sphinx-theme +tensorboard