diff --git a/pyproject.toml b/pyproject.toml index 783d6b2..daf4fea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ namespaces = true # ----------------------------------------- Project Metadata ------------------------------------- # [project] -version = "0.0.0.dev345" +version = "0.0.0.dev346" name = "ControlMan" dependencies = [ "packaging >= 23.2, < 24", @@ -33,7 +33,7 @@ dependencies = [ "GitTidy == 0.0.0.dev56", "PkgData == 0.0.0.dev5", "PyShellMan == 0.0.0.dev20", - "PySyntax == 0.0.0.dev4", + "PySyntax == 0.0.0.dev5", "ExceptionMan == 0.0.0.dev30", "MDit == 0.0.0.dev30", "JSONSchemata == 0.0.0.dev29", diff --git a/requirements.txt b/requirements.txt index fb29658..651f85e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ PySerials == 0.0.0.dev33 GitTidy == 0.0.0.dev56 PkgData == 0.0.0.dev5 PyShellMan == 0.0.0.dev20 -PySyntax == 0.0.0.dev4 +PySyntax == 0.0.0.dev5 ExceptionMan == 0.0.0.dev30 MDit == 0.0.0.dev30 JSONSchemata == 0.0.0.dev29 diff --git a/src/controlman/_data/schema/def/license-component.yaml b/src/controlman/_data/schema/def/license-component.yaml index 63b7001..2a5608d 100644 --- a/src/controlman/_data/schema/def/license-component.yaml +++ b/src/controlman/_data/schema/def/license-component.yaml @@ -1,11 +1,19 @@ $id: https://controlman.repodynamics.com/schema/license-component $schema: https://json-schema.org/draft/2020-12/schema -title: License Details +title: License Component type: object required: [ path ] properties: + type: + description: | + Type of the component, i.e., either `license` or `exception`. + type: string + enum: [ license, exception ] + custom: + description: | + Whether the component is a user-defined license or exception. + type: boolean path: - title: File Path description: | Path to the repository's [`LICENSE`](https://docs.github.com/en/communities/setting-up-your-project-for-healthy-contributions/adding-a-license-to-a-repository) diff --git a/src/controlman/_data/schema/def/pkg.yaml b/src/controlman/_data/schema/def/pkg.yaml index 029bed5..6c75b5b 100644 --- a/src/controlman/_data/schema/def/pkg.yaml +++ b/src/controlman/_data/schema/def/pkg.yaml @@ -554,13 +554,81 @@ properties: pip: path: ${{ ....path.root }}$/requirements.txt file: - title: Package documentation settings + description: | + Configurations for Python source files. type: object additionalProperties: + title: File Configurations + description: | + Configurations for a set of Python source files defined by a glob pattern. type: object + additionalProperties: false + required: [ glob ] + anyOf: + - required: [ docstring ] + - required: [ header_comments ] properties: + glob: + description: | + Glob pattern to match source files. + The pattern is relative to the [import package directory](#cccdef-pkg-path-import). + $ref: https://jsonschemata.repodynamics.com/string/oneline docstring: - type: string + description: | + Docstring configurations for the source files. + type: object + additionalProperties: false + required: [ mode, content ] + properties: + content: + description: | + Content of the docstring. + type: string + mode: + description: | + Mode of the docstring. + type: string + enum: [ append, prepend, replace ] + default: append + max_line_length: + description: | + Maximum line length of the docstring. + type: integer + minimum: 0 + header_comments: + description: | + Header comments for the source files. + type: object + additionalProperties: false + required: [ mode, content ] + properties: + mode: + description: | + Mode of the header comments. + type: string + enum: [ append, prepend, replace ] + default: append + content: + description: | + Content of the header comments. + type: string + max_line_length: + description: | + Maximum line length of the comments. + type: integer + minimum: 0 + line_continuation_indent: + description: | + Indentation of line continuation. + type: integer + minimum: 0 + default: 0 + empty_lines: + description: | + Number of empty lines between comments. + type: integer + minimum: 0 + default: 1 manifest: title: MANIFEST.in configurations. description: | diff --git a/src/controlman/_data/schema/main.yaml b/src/controlman/_data/schema/main.yaml index 01da442..3a4e387 100644 --- a/src/controlman/_data/schema/main.yaml +++ b/src/controlman/_data/schema/main.yaml @@ -34,10 +34,9 @@ description: | maintenance, testing, and refactoring tasks. type: object -required: [ repo, branch, control ] +required: [ name, repo, branch, control ] properties: name: - title: Name description: | Name of the project. @@ -74,7 +73,6 @@ properties: - My-Project $ref: https://jsonschemata.repodynamics.com/string/oneline title: - title: Title description: | A single-line description or slogan of the project. @@ -94,7 +92,6 @@ properties: - Python Tools for Quantum Mechanics Simulations and Research $ref: https://jsonschemata.repodynamics.com/string/oneline abstract: - title: Abstract description: | A short description of the project. It can contain GitHub Flavored Markdown and HTML syntax, @@ -129,7 +126,6 @@ properties: Cras eros risus, viverra ut lectus nec, lobortis rhoncus felis. $ref: https://jsonschemata.repodynamics.com/string/nonempty keywords: - title: Keywords description: | Keywords categorizing the project. @@ -148,7 +144,6 @@ properties: - Scientific Computing $ref: https://jsonschemata.repodynamics.com/array/unique-strings highlights: - title: Highlights description: | Key features of the project. @@ -189,7 +184,6 @@ properties: description: Description of the project feature. $ref: https://jsonschemata.repodynamics.com/string/nonempty license: - title: License description: | License of the project. You can either select one of the supported [SPDX](https://spdx.org/licenses/) licenses @@ -230,27 +224,45 @@ properties: required: [ expression ] properties: expression: - title: SPDX License Expression - description: | - [SPDX](https://spdx.org/licenses/) identifier of the license. - If one of the supported IDs are selected, - the rest of the keys are automatically filled in. - Otherwise, `name`, `text`, and `notice` must be provided. - Supported IDs are: - - - `AGPL-3.0-or-later`: [GNU Affero General Public License v3 or later](https://choosealicense.com/licenses/agpl-3.0/) - - `AGPL-3.0`: [GNU Affero General Public License v3](https://choosealicense.com/licenses/agpl-3.0/) - - `GPL-3.0-or-later`: [GNU General Public License v3 or later](https://choosealicense.com/licenses/gpl-3.0/) - - `GPL-3.0`: [GNU General Public License v3](https://choosealicense.com/licenses/gpl-3.0/) - - `BSD-3-Clause`: [BSD 3-Clause "New" or "Revised" License](https://choosealicense.com/licenses/bsd-3-clause/) - - `BSD-2-Clause`: [BSD 2-Clause "Simplified" License](https://choosealicense.com/licenses/bsd-2-clause/) - - `MIT`: [MIT License](https://choosealicense.com/licenses/mit/) - - `BSL-1.0`: [Boost Software License 1.0](https://choosealicense.com/licenses/bsl-1.0/) - - `Apache-2.0`: [Apache License 2.0](https://choosealicense.com/licenses/apache-2.0/) - - `MPL-2.0`: [Mozilla Public License 2.0](https://choosealicense.com/licenses/mpl-2.0/) - - `Unlicense`: [The Unlicense](https://choosealicense.com/licenses/unlicense/) $ref: https://jsonschemata.repodynamics.com/string/nonempty + description: | + An [SPDX license expression](https://spdx.github.io/spdx-spec/v3.0.1/annexes/spdx-license-expressions/) + defining the license of the project. + description_examples: | + To select a single license from the [SPDX License List](https://spdx.org/licenses/), + simply provide its [SPDX license ID](https://spdx.dev/learn/handling-license-info/). + Some common SPDX license IDs are: + - `AGPL-3.0-or-later`: [GNU Affero General Public License v3.0 or later](https://spdx.org/licenses/AGPL-3.0-or-later.html) + - `GPL-3.0-or-later`: [GNU General Public License v3.0 or later](https://spdx.org/licenses/GPL-3.0-or-later.html) + - `LGPL-3.0-or-later`: [GNU Lesser General Public License v3.0 or later](https://spdx.org/licenses/LGPL-3.0-or-later.html) + - `Apache-2.0`: [Apache License 2.0](https://spdx.org/licenses/Apache-2.0.html) + - `BSL-1.0`: [Boost Software License 1.0](https://spdx.org/licenses/BSL-1.0.html) + - `MIT`: [MIT License](https://spdx.org/licenses/MIT.html) + - `BSD-3-Clause`: [BSD 3-Clause "New" or "Revised" License](https://spdx.org/licenses/BSD-3-Clause.html) + - `BSD-2-Clause`: [BSD 2-Clause "Simplified" License](https://spdx.org/licenses/BSD-2-Clause.html) + - `MPL-2.0`: [Mozilla Public License 2.0](https://spdx.org/licenses/MPL-2.0.html) + - `Unlicense`: [The Unlicense](https://spdx.org/licenses/Unlicense.html) + [Composite licenses](https://spdx.github.io/spdx-spec/v3.0.1/annexes/spdx-license-expressions/#composite-license-expressions) + can be defined using "OR", "AND", and "WITH" operators with parentheses: + - `OR`: Allows either of the licenses to be used. + - `AND`: Requires compliance with both licenses. + - `WITH`: Defines an [exception](https://spdx.org/licenses/exceptions-index.html) to the license. + - `(` and `)`: Grouping of expressions. + For example: + - `MIT OR GPL-3.0-or-later` allows either the `MIT` or `GPL-3.0-or-later` license to be used. + - `LGPL-2.1-only OR MIT OR BSD-3-Clause` allows either the `LGPL-2.1-only`, `MIT`, or `BSD-3-Clause` license to be used. + - `LGPL-2.1-only AND MIT` requires compliance with both the `LGPL-2.1-only` and `MIT` licenses. + - `GPL-2.0-or-later WITH Bison-exception-2.2` adds `Bison-exception-2.2` to the `GPL-2.0-or-later` license. + - `MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)` requires compliance with the `MIT` license and either the `LGPL-2.1-or-later` or `BSD-3-Clause` license. + Custom licenses and exceptions can also be defined + using the `LicenseRef-` and `AdditionRef-` prefixes, respectively. + For example, to add a custom exception to the `MIT` license, + you can use `MIT WITH AdditionRef-My-Exception-1.0`. + The custom exception `AdditionRef-My-Exception-1.0` + must then be defined under [`$.license.component`](#ccc-license-component). component: + description: | + Information about each component of the license expression. type: object patternProperties: '^(DocumentRef-[a-zA-Z0-9-.]+:)?(AdditionRef|LicenseRef)-[a-zA-Z0-9.-]+$': @@ -272,23 +284,6 @@ properties: header: default: { } $ref: https://controlman.repodynamics.com/schema/license-component-config - header: - title: Header - description: | - Custom license notice. - $ref: https://jsonschemata.repodynamics.com/string/nonempty - path: - type: object - additionalProperties: false - properties: - texts_plain: - type: array - items: - $ref: https://jsonschemata.repodynamics.com/path/posix/absolute-from-cwd - headers_plain: - type: array - items: - $ref: https://jsonschemata.repodynamics.com/path/posix/absolute-from-cwd copyright: title: Copyright description: | @@ -3506,20 +3501,75 @@ properties: additionalProperties: false properties: duplicate: + description: | + Definition of duplicate locations for repository files. + Files defined here will be duplicated to the specified destinations + and dynamically kept in sync with the source. + + For each duplication, define a key-value pair where the key is an ID, + and the value is a mapping with keys `source` and `destinations`, + or `sources` and `destinations`. + description_examples: | + - Duplicate a file at `path/to/file.txt` to `path/to/backup/file.txt`: + ```yaml + backup_file: + source: path/to/file.txt + destinations: + - path/to/backup/file.txt + ``` + - Duplicate a file at `path/to/file.txt` to both + `path/to/backup1/backup_file.txt` and `path/to/backup2/backup_file.txt`: + ```yaml + backup_files: + source: path/to/file.txt + destinations: + - path/to/backup1/backup_file.txt + - path/to/backup2/backup_file.txt + - Duplicate multiple files at `path/to/file1.txt` and `path/to/file2.txt` to `path/to/backup_dir`: + ```yaml + backup_files: + sources: + - path/to/file1.txt + - path/to/file2.txt + destinations: + - path/to/backup_dir + - Duplicate all text files in `path/to/text_files` to `path/to/backup_dir`: + ```yaml + backup_files: + sources: + - path/to/text_files/*.txt + destinations: + - path/to/backup_dir + ``` type: object additionalProperties: + title: Duplicate Files type: object additionalProperties: false - required: [ destination ] + required: [ destinations ] properties: source: + description: | + Path to a single file to be duplicated. $ref: https://jsonschemata.repodynamics.com/path/posix/absolute-from-cwd sources: + description: | + Paths or glob patterns for multiple files to be duplicated. type: array + uniqueItems: true + items: + $ref: https://jsonschemata.repodynamics.com/path/posix/absolute-from-cwd + destinations: + description: | + Paths to the locations where the file(s) should be duplicated. + When used with `source`, each path must be a full path to a file (including the filename). + When used with `sources`, each path must be a directory path, + where all source files are copied to, keeping their original filenames. + type: array + uniqueItems: true + minItems: 1 items: $ref: https://jsonschemata.repodynamics.com/path/posix/absolute-from-cwd - destination: - $ref: https://jsonschemata.repodynamics.com/path/posix/absolute-from-cwd oneOf: - required: [ source ] - required: [ sources ] diff --git a/src/controlman/center_manager.py b/src/controlman/center_manager.py index bd3a0bd..b1ab781 100644 --- a/src/controlman/center_manager.py +++ b/src/controlman/center_manager.py @@ -208,18 +208,24 @@ def apply_changes(self) -> None: for duplicate in self._data_before.get("file.duplicate", {}).values(): if "source" in duplicate: - self._path_root.joinpath(duplicate["destination"]).unlink(missing_ok=True) + for destination in duplicate["destinations"]: + self._path_root.joinpath(destination).unlink(missing_ok=True) else: - for source in duplicate["sources"]: - self._path_root.joinpath(duplicate["destination"]).joinpath(_Path(source).stem).unlink(missing_ok=True) + for source_glob in duplicate["sources"]: + for source in self._path_root.glob(source_glob): + for destination in duplicate["destinations"]: + self._path_root.joinpath(destination).joinpath(_Path(source).stem).unlink(missing_ok=True) for duplicate in self._data.get("file.duplicate", {}).values(): if "source" in duplicate: - _shutil.copy2(self._path_root.joinpath(duplicate["source"]), self._path_root.joinpath(duplicate["destination"])) + for destination in duplicate["destinations"]: + _shutil.copy2(self._path_root.joinpath(duplicate["source"]), self._path_root.joinpath(destination)) else: - for source in duplicate["sources"]: - destination_path = self._path_root.joinpath(duplicate["destination"]).joinpath(_Path(source).stem) - destination_path.parent.mkdir(parents=True, exist_ok=True) - _shutil.copy2(self._path_root.joinpath(source), destination_path) + for source_glob in duplicate["sources"]: + for source in self._path_root.glob(source_glob): + for destination in duplicate["destinations"]: + destination_path = self._path_root.joinpath(destination).joinpath(_Path(source).stem) + destination_path.parent.mkdir(parents=True, exist_ok=True) + _shutil.copy2(self._path_root.joinpath(source), destination_path) return def _compare_dirs(self): diff --git a/src/controlman/data_gen/main.py b/src/controlman/data_gen/main.py index 6ea5e3a..dd8d76e 100644 --- a/src/controlman/data_gen/main.py +++ b/src/controlman/data_gen/main.py @@ -118,8 +118,6 @@ def _license(self): json_path="license.expression", data=self._data(), ) - path_texts = [] - path_headers = [] for spdx_ids, spdx_typ in ((license_ids, "license"), (exception_ids, "exception")): func = _spdx.license if spdx_typ == "license" else _spdx.exception class_ = _spdx.SPDXLicense if spdx_typ == "license" else _spdx.SPDXLicenseException @@ -177,11 +175,6 @@ def _license(self): raise_duplicates=False, raise_type_mismatch=True, ) - path_texts.append(path_text) - if header_xml: - path_headers.append(path_header) - self._data["license.path.texts_plain"] = path_texts - self._data["license.path.headers_plain"] = path_headers return def _discussion_categories(self): diff --git a/src/controlman/file_gen/python.py b/src/controlman/file_gen/python.py index 1232aa0..0eb7302 100644 --- a/src/controlman/file_gen/python.py +++ b/src/controlman/file_gen/python.py @@ -4,6 +4,7 @@ from typing import Literal import textwrap from pathlib import Path as _Path +import re as _re # Non-standard libraries import pyserials as _ps @@ -130,9 +131,9 @@ def python_files(self) -> list[DynamicFile]: # Get all file glob matches path_to_globs_map = {} abs_path = self._path_repo / (self._path_import_before or self._path_import) - for file_glob, file_config in self._pkg.get("file", {}).items(): - for filepath_match in abs_path.glob(file_glob): - path_to_globs_map.setdefault(filepath_match, []).append((file_glob, file_config)) + for config_id, file_config in self._pkg.get("file", {}).items(): + for filepath_match in abs_path.glob(file_config["glob"]): + path_to_globs_map.setdefault(filepath_match, []).append((config_id, file_config)) if not (mapping or path_to_globs_map): return [] # Process each file @@ -140,17 +141,25 @@ def python_files(self) -> list[DynamicFile]: for filepath in abs_path.glob("**/*.py"): file_content = filepath.read_text() if mapping: - file_content = _pysyntax.modify.rename_imports(module_content=file_content, mapping=mapping) + file_content = _pysyntax.modify.imports(module_content=file_content, mapping=mapping) if filepath in path_to_globs_map: - for matched_glob, file_config in path_to_globs_map[filepath]: + for config_id, file_config in path_to_globs_map[filepath]: if "docstring" in file_config: - docstring_before = self._pkg_before.get("file", {}).get(matched_glob, {}).get("docstring") + docstring_before = self._pkg_before.get("file", {}).get(config_id, {}).get("docstring") if docstring_before != file_config["docstring"]: file_content = self._update_docstring( file_content, file_config["docstring"], docstring_before, ) + if "header_comments" in file_config: + header_commens_before = self._pkg_before.get("file", {}).get(config_id, {}).get("header_comments") + if header_commens_before != file_config["header_comments"]: + file_content = self._update_header_comments( + file_content, + file_config["header_comments"], + header_commens_before, + ) subtype = filepath.relative_to(self._path_repo / self._path_src) subtype_display = str(subtype.with_suffix("")).replace("/", ".") fullpath_import_before = self._path_repo / (self._path_import_before or self._path_import) @@ -165,26 +174,85 @@ def python_files(self) -> list[DynamicFile]: ) return out - def _update_docstring(self, file_content: str, template: str, template_before: str) -> str: + def _update_docstring(self, file_content: str, template: dict, template_before: dict) -> str: - def get_wrapped_docstring(string): + def get_wrapped_docstring(templ: dict) -> str: + max_line_length = templ.get("max_line_length") + if not max_line_length: + return templ["content"] lines = [] - for line in string.strip().splitlines(): - line_parts = textwrap.wrap(line, width=80) + for line in templ["content"].splitlines(): + line_parts = textwrap.wrap(line, width=max_line_length, subsequent_indent=self._get_whitespace(line, leading=True)) lines.append('') if not line_parts else lines.extend(line_parts) - return "\n".join(lines), lines + wrapped_docstring = "\n".join(lines) + return f"{wrapped_docstring}{self._get_whitespace(templ['content'], leading=False)}" - docstring_text, docstring_lines = get_wrapped_docstring(template) + docstring_text = get_wrapped_docstring(template) docstring_before = _pysyntax.parse.docstring(file_content) - if docstring_before is None: - ending = '\n' if len(docstring_lines) > 1 else '' - docstring_replacement = f'{docstring_text}{ending}' + if template["mode"] == "replace" or docstring_before is None: + docstring_replacement = docstring_text + elif not template_before: + if template["mode"] == "prepend": + docstring_replacement = f"{docstring_text}{docstring_before}" + else: + docstring_replacement = f"{docstring_before}{docstring_text}" + else: + template_before_wrapped = get_wrapped_docstring(template_before) + docstring_replacement = docstring_before.replace(template_before_wrapped, "", 1) + if template["mode"] == "prepend": + docstring_replacement = f"{docstring_text}{docstring_replacement}" + else: + docstring_replacement = f"{docstring_replacement}{docstring_text}" + return _pysyntax.modify.docstring(file_content, docstring_replacement) + + + def _update_header_comments(self, file_content: str, template: dict, template_before: dict) -> str: + + def get_wrapped_header_comments(templ: dict) -> str: + max_line_length = templ.get("max_line_length") + lines = [] + current_newlines = 0 + for line in templ["content"].splitlines(): + if not line: + current_newlines += 1 + continue + if current_newlines: + if current_newlines == 2: + lines.append('#') + elif current_newlines > 2: + lines.append('') + current_newlines = 0 + if max_line_length: + line_indent = self._get_whitespace(line, leading=True) + line_parts = textwrap.wrap( + line, + width=max_line_length, + initial_indent=f"# {line_indent}", + subsequent_indent=f"# {line_indent}{templ['line_continuation_indent'] * " "}", + ) + else: + line_parts = [f"# {line}"] + lines.extend(line_parts) + return "\n".join(lines) + + header_comments_text = get_wrapped_header_comments(template) + header_comments_before = "\n".join(_pysyntax.parse.header_comments(file_content)) + newlines = "\n" * (template["empty_lines"] + 1) + if template["mode"] == "replace" or header_comments_before is None: + header_comments_replacement = header_comments_text elif not template_before: - docstring_replacement = f"{docstring_before.strip()}\n\n{docstring_text}\n" + if template["mode"] == "prepend": + header_comments_replacement = f"{header_comments_text}{newlines}{header_comments_before.strip()}" + else: + header_comments_replacement = f"{header_comments_before.strip()}{newlines}{header_comments_text}" else: - template_before_wrapped, _ = get_wrapped_docstring(template_before) - docstring_replacement = docstring_before.replace(template_before_wrapped, docstring_text, 1) - return _pysyntax.modify.update_docstring(file_content, docstring_replacement) + template_before_wrapped = get_wrapped_header_comments(template_before) + header_comments_replacement = header_comments_before.replace(template_before_wrapped, "", 1) + if template["mode"] == "prepend": + header_comments_replacement = f"{header_comments_text}{newlines}{header_comments_replacement.strip()}" + else: + header_comments_replacement = f"{header_comments_replacement.strip()}{newlines}{header_comments_text}" + return _pysyntax.modify.header_comments(file_content, header_comments_replacement) def manifest(self) -> list[DynamicFile]: if self.is_disabled("manifest"): @@ -304,3 +372,8 @@ def _scripts(self, typ: Literal["cli", "gui"]) -> dict[str, str]: for entry in self._data.get(f"{self._type}.entry.{typ}", {}).values(): scripts[entry["name"]] = entry["ref"] return scripts + + @staticmethod + def _get_whitespace(string: str, leading: bool) -> str: + match = _re.match(r"^\s*", string) if leading else _re.search(r"\s*$", string) + return match.group() if match else ""