nixos-render-docs: move options conversion to options module
This commit is contained in:
pennae
parent
aa3fd2865b
commit
ccb586299d
@ -16,129 +16,13 @@ from mdit_py_plugins.deflist import deflist_plugin
|
||||
from mdit_py_plugins.myst_role import myst_role_plugin
|
||||
from xml.sax.saxutils import escape, quoteattr
|
||||
|
||||
from .docbook import make_xml_id, DocBookRenderer
|
||||
from .md import md_escape
|
||||
from .options import option_is
|
||||
|
||||
class Converter:
|
||||
def __init__(self, manpage_urls: Dict[str, str]):
|
||||
self._manpage_urls = frozendict(manpage_urls)
|
||||
|
||||
self._md = markdown_it.MarkdownIt(
|
||||
"commonmark",
|
||||
{
|
||||
'maxNesting': 100, # default is 20
|
||||
'html': False, # not useful since we target many formats
|
||||
'typographer': True, # required for smartquotes
|
||||
},
|
||||
renderer_cls=lambda parser: DocBookRenderer(self._manpage_urls, parser)
|
||||
)
|
||||
# TODO maybe fork the plugin and have only a single rule for all?
|
||||
self._md.use(container_plugin, name="{.note}")
|
||||
self._md.use(container_plugin, name="{.important}")
|
||||
self._md.use(container_plugin, name="{.warning}")
|
||||
self._md.use(deflist_plugin)
|
||||
self._md.use(myst_role_plugin)
|
||||
self._md.enable(["smartquotes", "replacements"])
|
||||
|
||||
def render(self, src: str) -> str:
|
||||
return self._md.render(src)
|
||||
|
||||
md = Converter(json.load(open(os.getenv('MANPAGE_URLS'))))
|
||||
|
||||
# converts in-place!
|
||||
def convertMD(options: Dict[str, Any]) -> str:
|
||||
def convertCode(name: str, option: Dict[str, Any], key: str):
|
||||
if option_is(option, key, 'literalMD'):
|
||||
option[key] = md.render(f"*{key.capitalize()}:*\n{option[key]['text']}")
|
||||
elif option_is(option, key, 'literalExpression'):
|
||||
code = option[key]['text']
|
||||
# for multi-line code blocks we only have to count ` runs at the beginning
|
||||
# of a line, but this is much easier.
|
||||
multiline = '\n' in code
|
||||
longest, current = (0, 0)
|
||||
for c in code:
|
||||
current = current + 1 if c == '`' else 0
|
||||
longest = max(current, longest)
|
||||
# inline literals need a space to separate ticks from content, code blocks
|
||||
# need newlines. inline literals need one extra tick, code blocks need three.
|
||||
ticks, sep = ('`' * (longest + (3 if multiline else 1)), '\n' if multiline else ' ')
|
||||
code = f"{ticks}{sep}{code}{sep}{ticks}"
|
||||
option[key] = md.render(f"*{key.capitalize()}:*\n{code}")
|
||||
elif option_is(option, key, 'literalDocBook'):
|
||||
option[key] = f"<para><emphasis>{key.capitalize()}:</emphasis> {option[key]['text']}</para>"
|
||||
elif key in option:
|
||||
raise Exception(f"{name} {key} has unrecognized type", option[key])
|
||||
|
||||
for (name, option) in options.items():
|
||||
try:
|
||||
if option_is(option, 'description', 'mdDoc'):
|
||||
option['description'] = md.render(option['description']['text'])
|
||||
elif markdownByDefault:
|
||||
option['description'] = md.render(option['description'])
|
||||
else:
|
||||
option['description'] = ("<nixos:option-description><para>" +
|
||||
option['description'] +
|
||||
"</para></nixos:option-description>")
|
||||
|
||||
convertCode(name, option, 'example')
|
||||
convertCode(name, option, 'default')
|
||||
|
||||
if typ := option.get('type'):
|
||||
ro = " *(read only)*" if option.get('readOnly', False) else ""
|
||||
option['type'] = md.render(f'*Type:* {md_escape(typ)}{ro}')
|
||||
|
||||
if 'relatedPackages' in option:
|
||||
option['relatedPackages'] = md.render(option['relatedPackages'])
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to render option {name}") from e
|
||||
|
||||
return options
|
||||
from .options import DocBookConverter
|
||||
|
||||
def need_env(n):
|
||||
if n not in os.environ:
|
||||
raise RuntimeError("required environment variable not set", n)
|
||||
return os.environ[n]
|
||||
|
||||
OTD_REVISION = need_env('OTD_REVISION')
|
||||
OTD_DOCUMENT_TYPE = need_env('OTD_DOCUMENT_TYPE')
|
||||
OTD_VARIABLE_LIST_ID = need_env('OTD_VARIABLE_LIST_ID')
|
||||
OTD_OPTION_ID_PREFIX = need_env('OTD_OPTION_ID_PREFIX')
|
||||
|
||||
def print_decl_def(header, locs):
|
||||
print(f"""<para><emphasis>{header}:</emphasis></para>""")
|
||||
print(f"""<simplelist>""")
|
||||
for loc in locs:
|
||||
# locations can be either plain strings (specific to nixpkgs), or attrsets
|
||||
# { name = "foo/bar.nix"; url = "https://github.com/....."; }
|
||||
if isinstance(loc, str):
|
||||
# Hyperlink the filename either to the NixOS github
|
||||
# repository (if it’s a module and we have a revision number),
|
||||
# or to the local filesystem.
|
||||
if not loc.startswith('/'):
|
||||
if OTD_REVISION == 'local':
|
||||
href = f"https://github.com/NixOS/nixpkgs/blob/master/{loc}"
|
||||
else:
|
||||
href = f"https://github.com/NixOS/nixpkgs/blob/{OTD_REVISION}/{loc}"
|
||||
else:
|
||||
href = f"file://{loc}"
|
||||
# Print the filename and make it user-friendly by replacing the
|
||||
# /nix/store/<hash> prefix by the default location of nixos
|
||||
# sources.
|
||||
if not loc.startswith('/'):
|
||||
name = f"<nixpkgs/{loc}>"
|
||||
elif loc.contains('nixops') and loc.contains('/nix/'):
|
||||
name = f"<nixops/{loc[loc.find('/nix/') + 5:]}>"
|
||||
else:
|
||||
name = loc
|
||||
print(f"""<member><filename xlink:href={quoteattr(href)}>""")
|
||||
print(escape(name))
|
||||
print(f"""</filename></member>""")
|
||||
else:
|
||||
href = f" xlink:href={quoteattr(loc['url'])}" if 'url' in loc else ""
|
||||
print(f"""<member><filename{href}>{escape(loc['name'])}</filename></member>""")
|
||||
print(f"""</simplelist>""")
|
||||
|
||||
def main():
|
||||
markdownByDefault = False
|
||||
optOffset = 0
|
||||
@ -147,49 +31,15 @@ def main():
|
||||
optOffset += 1
|
||||
markdownByDefault = True
|
||||
|
||||
options = convertMD(json.load(open(sys.argv[1 + optOffset], 'r')))
|
||||
md = DocBookConverter(
|
||||
json.load(open(os.getenv('MANPAGE_URLS'))),
|
||||
revision = need_env('OTD_REVISION'),
|
||||
document_type = need_env('OTD_DOCUMENT_TYPE'),
|
||||
varlist_id = need_env('OTD_VARIABLE_LIST_ID'),
|
||||
id_prefix = need_env('OTD_OPTION_ID_PREFIX'),
|
||||
markdown_by_default = markdownByDefault
|
||||
)
|
||||
|
||||
keys = list(options.keys())
|
||||
keys.sort(key=lambda opt: [ (0 if p.startswith("enable") else 1 if p.startswith("package") else 2, p)
|
||||
for p in options[opt]['loc'] ])
|
||||
|
||||
print(f"""<?xml version="1.0" encoding="UTF-8"?>""")
|
||||
if OTD_DOCUMENT_TYPE == 'appendix':
|
||||
print("""<appendix xmlns="http://docbook.org/ns/docbook" xml:id="appendix-configuration-options">""")
|
||||
print(""" <title>Configuration Options</title>""")
|
||||
print(f"""<variablelist xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:nixos="tag:nixos.org"
|
||||
xmlns="http://docbook.org/ns/docbook"
|
||||
xml:id="{OTD_VARIABLE_LIST_ID}">""")
|
||||
|
||||
for name in keys:
|
||||
opt = options[name]
|
||||
id = OTD_OPTION_ID_PREFIX + make_xml_id(name)
|
||||
print(f"""<varlistentry>""")
|
||||
# NOTE adding extra spaces here introduces spaces into xref link expansions
|
||||
print(f"""<term xlink:href={quoteattr("#" + id)} xml:id={quoteattr(id)}>""", end='')
|
||||
print(f"""<option>{escape(name)}</option>""", end='')
|
||||
print(f"""</term>""")
|
||||
print(f"""<listitem>""")
|
||||
print(opt['description'])
|
||||
if typ := opt.get('type'):
|
||||
print(typ)
|
||||
if default := opt.get('default'):
|
||||
print(default)
|
||||
if example := opt.get('example'):
|
||||
print(example)
|
||||
if related := opt.get('relatedPackages'):
|
||||
print(f"""<para>""")
|
||||
print(f""" <emphasis>Related packages:</emphasis>""")
|
||||
print(f"""</para>""")
|
||||
print(related)
|
||||
if decl := opt.get('declarations'):
|
||||
print_decl_def("Declared by", decl)
|
||||
if defs := opt.get('definitions'):
|
||||
print_decl_def("Defined by", defs)
|
||||
print(f"""</listitem>""")
|
||||
print(f"""</varlistentry>""")
|
||||
|
||||
print("""</variablelist>""")
|
||||
if OTD_DOCUMENT_TYPE == 'appendix':
|
||||
print("""</appendix>""")
|
||||
options = json.load(open(sys.argv[1 + optOffset], 'r'))
|
||||
md.add_options(options)
|
||||
print(md.finalize())
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from typing import Any, Optional
|
||||
from frozendict import frozendict # type: ignore[attr-defined]
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import markdown_it
|
||||
from markdown_it.token import Token
|
||||
from markdown_it.utils import OptionsDict
|
||||
from mdit_py_plugins.container import container_plugin # type: ignore[attr-defined]
|
||||
from mdit_py_plugins.deflist import deflist_plugin # type: ignore[attr-defined]
|
||||
from mdit_py_plugins.myst_role import myst_role_plugin # type: ignore[attr-defined]
|
||||
|
||||
_md_escape_table = {
|
||||
ord('*'): '\\*',
|
||||
@ -175,3 +179,29 @@ class Renderer(markdown_it.renderer.RendererProtocol):
|
||||
def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
|
||||
env: MutableMapping[str, Any]) -> str:
|
||||
raise RuntimeError("md token not supported", token)
|
||||
|
||||
class Converter:
|
||||
__renderer__: Callable[[Mapping[str, str], markdown_it.MarkdownIt], Renderer]
|
||||
|
||||
def __init__(self, manpage_urls: Mapping[str, str]):
|
||||
self._manpage_urls = frozendict(manpage_urls)
|
||||
|
||||
self._md = markdown_it.MarkdownIt(
|
||||
"commonmark",
|
||||
{
|
||||
'maxNesting': 100, # default is 20
|
||||
'html': False, # not useful since we target many formats
|
||||
'typographer': True, # required for smartquotes
|
||||
},
|
||||
renderer_cls=lambda parser: self.__renderer__(self._manpage_urls, parser)
|
||||
)
|
||||
# TODO maybe fork the plugin and have only a single rule for all?
|
||||
self._md.use(container_plugin, name="{.note}")
|
||||
self._md.use(container_plugin, name="{.important}")
|
||||
self._md.use(container_plugin, name="{.warning}")
|
||||
self._md.use(deflist_plugin)
|
||||
self._md.use(myst_role_plugin)
|
||||
self._md.enable(["smartquotes", "replacements"])
|
||||
|
||||
def _render(self, src: str) -> str:
|
||||
return self._md.render(src) # type: ignore[no-any-return]
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
from xml.sax.saxutils import escape, quoteattr
|
||||
|
||||
from .types import Option
|
||||
from .docbook import DocBookRenderer, make_xml_id
|
||||
from .md import Converter, md_escape
|
||||
from .types import OptionLoc, Option, RenderedOption
|
||||
|
||||
def option_is(option: Option, key: str, typ: str) -> Optional[dict[str, str]]:
|
||||
if key not in option:
|
||||
@ -10,3 +13,187 @@ def option_is(option: Option, key: str, typ: str) -> Optional[dict[str, str]]:
|
||||
if option[key].get('_type') != typ: # type: ignore[union-attr]
|
||||
return None
|
||||
return option[key] # type: ignore[return-value]
|
||||
|
||||
class DocBookConverter(Converter):
|
||||
__renderer__ = DocBookRenderer
|
||||
_options: dict[str, RenderedOption]
|
||||
|
||||
def __init__(self, manpage_urls: dict[str, str],
|
||||
revision: str,
|
||||
document_type: str,
|
||||
varlist_id: str,
|
||||
id_prefix: str,
|
||||
markdown_by_default: bool):
|
||||
super().__init__(manpage_urls)
|
||||
self._options = {}
|
||||
self._revision = revision
|
||||
self._document_type = document_type
|
||||
self._varlist_id = varlist_id
|
||||
self._id_prefix = id_prefix
|
||||
self._markdown_by_default = markdown_by_default
|
||||
|
||||
def _format_decl_def_loc(self, loc: OptionLoc) -> tuple[Optional[str], str]:
|
||||
# locations can be either plain strings (specific to nixpkgs), or attrsets
|
||||
# { name = "foo/bar.nix"; url = "https://github.com/....."; }
|
||||
if isinstance(loc, str):
|
||||
# Hyperlink the filename either to the NixOS github
|
||||
# repository (if it’s a module and we have a revision number),
|
||||
# or to the local filesystem.
|
||||
if not loc.startswith('/'):
|
||||
if self._revision == 'local':
|
||||
href = f"https://github.com/NixOS/nixpkgs/blob/master/{loc}"
|
||||
else:
|
||||
href = f"https://github.com/NixOS/nixpkgs/blob/{self._revision}/{loc}"
|
||||
else:
|
||||
href = f"file://{loc}"
|
||||
# Print the filename and make it user-friendly by replacing the
|
||||
# /nix/store/<hash> prefix by the default location of nixos
|
||||
# sources.
|
||||
if not loc.startswith('/'):
|
||||
name = f"<nixpkgs/{loc}>"
|
||||
elif 'nixops' in loc and '/nix/' in loc:
|
||||
name = f"<nixops/{loc[loc.find('/nix/') + 5:]}>"
|
||||
else:
|
||||
name = loc
|
||||
return (href, name)
|
||||
else:
|
||||
return (loc['url'] if 'url' in loc else None, loc['name'])
|
||||
|
||||
def _render_decl_def(self, header: str, locs: list[OptionLoc]) -> list[str]:
|
||||
result = []
|
||||
result += self._decl_def_header(header)
|
||||
for loc in locs:
|
||||
href, name = self._format_decl_def_loc(loc)
|
||||
result += self._decl_def_entry(href, name)
|
||||
result += self._decl_def_footer()
|
||||
return result
|
||||
|
||||
def _render_code(self, option: Option, key: str) -> list[str]:
|
||||
if lit := option_is(option, key, 'literalDocBook'):
|
||||
return [ f"<para><emphasis>{key.capitalize()}:</emphasis> {lit['text']}</para>" ]
|
||||
elif lit := option_is(option, key, 'literalMD'):
|
||||
return [ self._render(f"*{key.capitalize()}:*\n{lit['text']}") ]
|
||||
elif lit := option_is(option, key, 'literalExpression'):
|
||||
code = lit['text']
|
||||
# for multi-line code blocks we only have to count ` runs at the beginning
|
||||
# of a line, but this is much easier.
|
||||
multiline = '\n' in code
|
||||
longest, current = (0, 0)
|
||||
for c in code:
|
||||
current = current + 1 if c == '`' else 0
|
||||
longest = max(current, longest)
|
||||
# inline literals need a space to separate ticks from content, code blocks
|
||||
# need newlines. inline literals need one extra tick, code blocks need three.
|
||||
ticks, sep = ('`' * (longest + (3 if multiline else 1)), '\n' if multiline else ' ')
|
||||
code = f"{ticks}{sep}{code}{sep}{ticks}"
|
||||
return [ self._render(f"*{key.capitalize()}:*\n{code}") ]
|
||||
elif key in option:
|
||||
raise Exception(f"{key} has unrecognized type", option[key])
|
||||
else:
|
||||
return []
|
||||
|
||||
def _render_description(self, desc: str | dict[str, str]) -> list[str]:
|
||||
if isinstance(desc, str) and not self._markdown_by_default:
|
||||
return [ f"<nixos:option-description><para>{desc}</para></nixos:option-description>" ]
|
||||
elif isinstance(desc, str) and self._markdown_by_default:
|
||||
return [ self._render(desc) ]
|
||||
elif isinstance(desc, dict) and desc.get('_type') == 'mdDoc':
|
||||
return [ self._render(desc['text']) ]
|
||||
else:
|
||||
raise Exception("description has unrecognized type", desc)
|
||||
|
||||
def _convert_one(self, option: dict[str, Any]) -> list[str]:
|
||||
result = []
|
||||
|
||||
if desc := option.get('description'):
|
||||
result += self._render_description(desc)
|
||||
if typ := option.get('type'):
|
||||
ro = " *(read only)*" if option.get('readOnly', False) else ""
|
||||
result.append(self._render(f"*Type:* {md_escape(typ)}{ro}"))
|
||||
|
||||
result += self._render_code(option, 'default')
|
||||
result += self._render_code(option, 'example')
|
||||
|
||||
if related := option.get('relatedPackages'):
|
||||
result += self._related_packages_header()
|
||||
result.append(self._render(related))
|
||||
if decl := option.get('declarations'):
|
||||
result += self._render_decl_def("Declared by", decl)
|
||||
if defs := option.get('definitions'):
|
||||
result += self._render_decl_def("Defined by", defs)
|
||||
|
||||
return result
|
||||
|
||||
def add_options(self, options: dict[str, Any]) -> None:
|
||||
for (name, option) in options.items():
|
||||
try:
|
||||
self._options[name] = RenderedOption(option['loc'], self._convert_one(option))
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to render option {name}") from e
|
||||
|
||||
def _related_packages_header(self) -> list[str]:
|
||||
return [
|
||||
"<para>",
|
||||
" <emphasis>Related packages:</emphasis>",
|
||||
"</para>",
|
||||
]
|
||||
|
||||
def _decl_def_header(self, header: str) -> list[str]:
|
||||
return [
|
||||
f"<para><emphasis>{header}:</emphasis></para>",
|
||||
"<simplelist>"
|
||||
]
|
||||
|
||||
def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]:
|
||||
if href is not None:
|
||||
href = " xlink:href=" + quoteattr(href)
|
||||
return [
|
||||
f"<member><filename{href}>",
|
||||
escape(name),
|
||||
"</filename></member>"
|
||||
]
|
||||
|
||||
def _decl_def_footer(self) -> list[str]:
|
||||
return [ "</simplelist>" ]
|
||||
|
||||
def finalize(self) -> str:
|
||||
keys = list(self._options.keys())
|
||||
keys.sort(key=lambda opt: [ (0 if p.startswith("enable") else 1 if p.startswith("package") else 2, p)
|
||||
for p in self._options[opt].loc ])
|
||||
|
||||
result = []
|
||||
|
||||
result.append('<?xml version="1.0" encoding="UTF-8"?>')
|
||||
if self._document_type == 'appendix':
|
||||
result += [
|
||||
'<appendix xmlns="http://docbook.org/ns/docbook"',
|
||||
' xml:id="appendix-configuration-options">',
|
||||
' <title>Configuration Options</title>',
|
||||
]
|
||||
result += [
|
||||
f'<variablelist xmlns:xlink="http://www.w3.org/1999/xlink"',
|
||||
' xmlns:nixos="tag:nixos.org"',
|
||||
' xmlns="http://docbook.org/ns/docbook"',
|
||||
f' xml:id="{self._varlist_id}">',
|
||||
]
|
||||
|
||||
for name in keys:
|
||||
id = make_xml_id(self._id_prefix + name)
|
||||
result += [
|
||||
"<varlistentry>",
|
||||
# NOTE adding extra spaces here introduces spaces into xref link expansions
|
||||
(f"<term xlink:href={quoteattr('#' + id)} xml:id={quoteattr(id)}>" +
|
||||
f"<option>{escape(name)}</option></term>"),
|
||||
"<listitem>"
|
||||
]
|
||||
result += self._options[name].lines
|
||||
result += [
|
||||
"</listitem>",
|
||||
"</varlistentry>"
|
||||
]
|
||||
|
||||
result.append("</variablelist>")
|
||||
if self._document_type == 'appendix':
|
||||
result.append("</appendix>")
|
||||
|
||||
return "\n".join(result)
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, Tuple, NamedTuple
|
||||
|
||||
OptionLoc = str | dict[str, str]
|
||||
Option = dict[str, str | dict[str, str] | list[OptionLoc]]
|
||||
|
||||
RenderedOption = NamedTuple('RenderedOption', [('loc', list[str]),
|
||||
('lines', list[str])])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user