catalog

trestle.core.commands.author.catalog

Author commands to generate catalog controls as markdown and assemble them back to json.

Attributes

logger = logging.getLogger(__name__) module-attribute

Classes

CatalogAssemble

Bases: AuthorCommonCommand

Assemble markdown files of controls into a Catalog json file.

Source code in trestle/core/commands/author/catalog.py

class CatalogAssemble(AuthorCommonCommand):
    """Assemble markdown files of controls into a Catalog json file."""

    name = 'catalog-assemble'

    def _init_arguments(self) -> None:
        name_help_str = (
            'Optional name of the catalog model in the trestle workspace that is being modified.  '
            'If not provided the output name is used.'
        )
        self.add_argument('-n', '--name', help=name_help_str, required=False, type=str)
        file_help_str = 'Name of the input markdown file directory'
        self.add_argument('-m', '--markdown', help=file_help_str, required=True, type=str)
        output_help_str = 'Name of the output generated json Catalog'
        self.add_argument('-o', '--output', help=output_help_str, required=True, type=str)
        self.add_argument('-sp', '--set-parameters', action='store_true', help=const.HELP_SET_PARAMS)
        self.add_argument('-r', '--regenerate', action='store_true', help=const.HELP_REGENERATE)
        self.add_argument('-vn', '--version', help=const.HELP_VERSION, required=False, type=str)

    def _run(self, args: argparse.Namespace) -> int:
        try:
            log.set_log_level_from_args(args)
            trestle_root = pathlib.Path(args.trestle_root)
            return CatalogAssemble.assemble_catalog(
                trestle_root=trestle_root,
                md_name=args.markdown,
                assem_cat_name=args.output,
                parent_cat_name=args.name,
                set_parameters_flag=args.set_parameters,
                regenerate=args.regenerate,
                version=args.version
            )
        except Exception as e:  # pragma: no cover
            return handle_generic_command_exception(e, logger, 'Error occurred while assembling catalog')

    @staticmethod
    def assemble_catalog(
        trestle_root: pathlib.Path,
        md_name: str,
        assem_cat_name: str,
        parent_cat_name: Optional[str],
        set_parameters_flag: bool,
        regenerate: bool,
        version: Optional[str]
    ) -> int:
        """
        Assemble the markdown directory into a json catalog model file.

        Args:
            trestle_root: The trestle root directory
            md_name: The name of the directory containing the markdown control files for the ssp
            assem_cat_name: The output name of the catalog model to be created from the assembly
            parent_cat_name: Optional name of the parent catalog that the markdown controls will replace
            set_parameters_flag: set the parameters and props in the control to the values in the markdown yaml header
            regenerate: whether to regenerate the uuid's in the catalog
            version: version for the assembled catalog

        Returns:
            0 on success, 1 otherwise

        Notes:
            If the destination catalog_name model already exists in the trestle workspace, it is overwritten.
            If a parent catalog is not specified, the assembled catalog will be used as the parent if it exists.
            If no parent catalog name is available, the catalog is created anew using only the markdown content.
        """
        md_dir = trestle_root / md_name
        if not md_dir.exists():
            raise TrestleError(f'Markdown directory {md_name} does not exist.')

        # assemble the markdown controls into fresh md_catalog
        catalog_api_from_md = CatalogAPI(catalog=None)
        try:
            md_catalog = catalog_api_from_md.read_catalog_from_markdown(md_dir, set_parameters_flag)
        except Exception as e:
            raise TrestleError(f'Error reading catalog from markdown {md_dir}: {e}')

        # this is None if it doesn't exist yet
        assem_cat_path = ModelUtils.get_model_path_for_name_and_class(trestle_root, assem_cat_name, Catalog)
        logger.debug(f'assem_cat_path is {assem_cat_path}')

        # if original cat is not specified, use the assembled cat but only if it already exists
        if not parent_cat_name and assem_cat_path:
            parent_cat_name = assem_cat_name

        # default to JSON but allow override later if other file type found
        new_content_type = FileContentType.JSON

        # if we have parent catalog then merge the markdown controls into it
        # the parent can be a separate catalog or the destination assembled catalog if it exists
        # but this is the catalog that the markdown is merged into in memory
        logger.debug(f'parent_cat_name is {parent_cat_name}')
        if parent_cat_name:
            parent_cat, parent_cat_path = load_validate_model_name(trestle_root, parent_cat_name, Catalog)
            parent_cat_api = CatalogAPI(catalog=parent_cat)
            # merge the just-read md catalog into the original json
            parent_cat_api.merge_catalog(md_catalog, set_parameters_flag)
            md_catalog = parent_cat_api._catalog_interface.get_catalog()
            new_content_type = FileContentType.path_to_content_type(parent_cat_path)

        if version:
            md_catalog.metadata.version = version

        # now check the destination catalog to see if the in-memory catalog matches it
        if assem_cat_path:
            new_content_type = FileContentType.path_to_content_type(assem_cat_path)
            existing_cat = load_validate_model_path(trestle_root, assem_cat_path)
            if ModelUtils.models_are_equivalent(existing_cat, md_catalog):  # type: ignore
                logger.info('Assembled catalog is not different from existing version, so no update.')
                return CmdReturnCodes.SUCCESS.value
            else:
                logger.debug('new assembled catalog is different from existing one')

        if regenerate:
            md_catalog, _, _ = ModelUtils.regenerate_uuids(md_catalog)
            logger.debug('regenerating uuids in catalog')
        ModelUtils.update_last_modified(md_catalog)

        md_catalog.metadata.oscal_version = OSCAL_VERSION

        # we still may not know the assem_cat_path but can now create it with file content type
        assem_cat_path = ModelUtils.get_model_path_for_name_and_class(
            trestle_root, assem_cat_name, Catalog, new_content_type
        )

        if assem_cat_path.parent.exists():
            logger.info('Creating catalog from markdown and destination catalog exists, so updating.')
            shutil.rmtree(str(assem_cat_path.parent))

        assem_cat_path.parent.mkdir(parents=True, exist_ok=True)
        md_catalog.oscal_write(assem_cat_path.parent / 'catalog.json')
        return CmdReturnCodes.SUCCESS.value

Attributes

name = 'catalog-assemble' class-attribute instance-attribute

Functions

assemble_catalog(trestle_root, md_name, assem_cat_name, parent_cat_name, set_parameters_flag, regenerate, version) staticmethod

Assemble the markdown directory into a json catalog model file.

Parameters:

Name	Type	Description	Default
trestle_root	Path	The trestle root directory	required
md_name	str	The name of the directory containing the markdown control files for the ssp	required
assem_cat_name	str	The output name of the catalog model to be created from the assembly	required
parent_cat_name	Optional[str]	Optional name of the parent catalog that the markdown controls will replace	required
set_parameters_flag	bool	set the parameters and props in the control to the values in the markdown yaml header	required
regenerate	bool	whether to regenerate the uuid's in the catalog	required
version	Optional[str]	version for the assembled catalog	required

Returns:

Type	Description
int	0 on success, 1 otherwise

Notes

If the destination catalog_name model already exists in the trestle workspace, it is overwritten. If a parent catalog is not specified, the assembled catalog will be used as the parent if it exists. If no parent catalog name is available, the catalog is created anew using only the markdown content.

Source code in trestle/core/commands/author/catalog.py

@staticmethod
def assemble_catalog(
    trestle_root: pathlib.Path,
    md_name: str,
    assem_cat_name: str,
    parent_cat_name: Optional[str],
    set_parameters_flag: bool,
    regenerate: bool,
    version: Optional[str]
) -> int:
    """
    Assemble the markdown directory into a json catalog model file.

    Args:
        trestle_root: The trestle root directory
        md_name: The name of the directory containing the markdown control files for the ssp
        assem_cat_name: The output name of the catalog model to be created from the assembly
        parent_cat_name: Optional name of the parent catalog that the markdown controls will replace
        set_parameters_flag: set the parameters and props in the control to the values in the markdown yaml header
        regenerate: whether to regenerate the uuid's in the catalog
        version: version for the assembled catalog

    Returns:
        0 on success, 1 otherwise

    Notes:
        If the destination catalog_name model already exists in the trestle workspace, it is overwritten.
        If a parent catalog is not specified, the assembled catalog will be used as the parent if it exists.
        If no parent catalog name is available, the catalog is created anew using only the markdown content.
    """
    md_dir = trestle_root / md_name
    if not md_dir.exists():
        raise TrestleError(f'Markdown directory {md_name} does not exist.')

    # assemble the markdown controls into fresh md_catalog
    catalog_api_from_md = CatalogAPI(catalog=None)
    try:
        md_catalog = catalog_api_from_md.read_catalog_from_markdown(md_dir, set_parameters_flag)
    except Exception as e:
        raise TrestleError(f'Error reading catalog from markdown {md_dir}: {e}')

    # this is None if it doesn't exist yet
    assem_cat_path = ModelUtils.get_model_path_for_name_and_class(trestle_root, assem_cat_name, Catalog)
    logger.debug(f'assem_cat_path is {assem_cat_path}')

    # if original cat is not specified, use the assembled cat but only if it already exists
    if not parent_cat_name and assem_cat_path:
        parent_cat_name = assem_cat_name

    # default to JSON but allow override later if other file type found
    new_content_type = FileContentType.JSON

    # if we have parent catalog then merge the markdown controls into it
    # the parent can be a separate catalog or the destination assembled catalog if it exists
    # but this is the catalog that the markdown is merged into in memory
    logger.debug(f'parent_cat_name is {parent_cat_name}')
    if parent_cat_name:
        parent_cat, parent_cat_path = load_validate_model_name(trestle_root, parent_cat_name, Catalog)
        parent_cat_api = CatalogAPI(catalog=parent_cat)
        # merge the just-read md catalog into the original json
        parent_cat_api.merge_catalog(md_catalog, set_parameters_flag)
        md_catalog = parent_cat_api._catalog_interface.get_catalog()
        new_content_type = FileContentType.path_to_content_type(parent_cat_path)

    if version:
        md_catalog.metadata.version = version

    # now check the destination catalog to see if the in-memory catalog matches it
    if assem_cat_path:
        new_content_type = FileContentType.path_to_content_type(assem_cat_path)
        existing_cat = load_validate_model_path(trestle_root, assem_cat_path)
        if ModelUtils.models_are_equivalent(existing_cat, md_catalog):  # type: ignore
            logger.info('Assembled catalog is not different from existing version, so no update.')
            return CmdReturnCodes.SUCCESS.value
        else:
            logger.debug('new assembled catalog is different from existing one')

    if regenerate:
        md_catalog, _, _ = ModelUtils.regenerate_uuids(md_catalog)
        logger.debug('regenerating uuids in catalog')
    ModelUtils.update_last_modified(md_catalog)

    md_catalog.metadata.oscal_version = OSCAL_VERSION

    # we still may not know the assem_cat_path but can now create it with file content type
    assem_cat_path = ModelUtils.get_model_path_for_name_and_class(
        trestle_root, assem_cat_name, Catalog, new_content_type
    )

    if assem_cat_path.parent.exists():
        logger.info('Creating catalog from markdown and destination catalog exists, so updating.')
        shutil.rmtree(str(assem_cat_path.parent))

    assem_cat_path.parent.mkdir(parents=True, exist_ok=True)
    md_catalog.oscal_write(assem_cat_path.parent / 'catalog.json')
    return CmdReturnCodes.SUCCESS.value

CatalogGenerate

Bases: AuthorCommonCommand

Generate Catalog controls in markdown form from a catalog in the trestle workspace.

Source code in trestle/core/commands/author/catalog.py

class CatalogGenerate(AuthorCommonCommand):
    """Generate Catalog controls in markdown form from a catalog in the trestle workspace."""

    name = 'catalog-generate'

    def _init_arguments(self) -> None:
        name_help_str = 'Name of the catalog model in the trestle workspace'
        self.add_argument('-n', '--name', help=name_help_str, required=True, type=str)
        self.add_argument(
            '-o', '--output', help='Name of the output generated catalog markdown folder', required=True, type=str
        )  # noqa E501
        self.add_argument('-fo', '--force-overwrite', help=const.HELP_FO_OUTPUT, required=False, action='store_true')
        self.add_argument('-y', '--yaml-header', help=const.HELP_YAML_PATH, required=False, type=str)
        self.add_argument(
            '-ohv',
            '--overwrite-header-values',
            help=const.HELP_OVERWRITE_HEADER_VALUES,
            required=False,
            action='store_true',
            default=False
        )

    def _run(self, args: argparse.Namespace) -> int:
        try:
            log.set_log_level_from_args(args)
            trestle_root = args.trestle_root
            if not file_utils.is_directory_name_allowed(args.output):
                raise TrestleError(f'{args.output} is not an allowed directory name')

            if args.force_overwrite:
                try:
                    logger.info(f'Overwriting the content in {args.output} folder.')
                    clear_folder(pathlib.Path(args.output))
                except TrestleError as e:  # pragma: no cover
                    raise TrestleError(f'Unable to overwrite contents in {args.output} folder: {e}')

            yaml_header: Dict[str, Any] = {}
            if args.yaml_header:
                try:
                    logging.debug(f'Loading yaml header file {args.yaml_header}')
                    yaml = YAML(typ='safe')
                    yaml_header = yaml.load(pathlib.Path(args.yaml_header).open('r'))
                except YAMLError as e:
                    raise TrestleError(f'YAML error loading yaml header {args.yaml_header} for ssp generation: {e}')

            catalog_path = trestle_root / f'catalogs/{args.name}/catalog.json'

            markdown_path = trestle_root / args.output

            return self.generate_markdown(
                trestle_root, catalog_path, markdown_path, yaml_header, args.overwrite_header_values
            )
        except Exception as e:  # pragma: no cover
            return handle_generic_command_exception(e, logger, 'Error occurred when generating markdown for catalog')

    def generate_markdown(
        self,
        trestle_root: pathlib.Path,
        catalog_path: pathlib.Path,
        markdown_path: pathlib.Path,
        yaml_header: Dict[str, Any],
        overwrite_header_values: bool
    ) -> int:
        """Generate markdown for the controls in the catalog."""
        try:
            catalog = load_validate_model_path(trestle_root, catalog_path)
            context = ControlContext.generate(
                ContextPurpose.CATALOG,
                True,
                trestle_root,
                markdown_path,
                cli_yaml_header=yaml_header,
                overwrite_header_values=overwrite_header_values,
                set_parameters_flag=True
            )
            catalog_api = CatalogAPI(catalog=catalog, context=context)
            catalog_api.write_catalog_as_markdown()

        except TrestleNotFoundError as e:
            raise TrestleError(f'Catalog {catalog_path} not found for load: {e}')
        except Exception as e:
            raise TrestleError(f'Error generating markdown for controls in {catalog_path}: {e}')

        return CmdReturnCodes.SUCCESS.value

Attributes

name = 'catalog-generate' class-attribute instance-attribute

Functions

generate_markdown(trestle_root, catalog_path, markdown_path, yaml_header, overwrite_header_values)

Generate markdown for the controls in the catalog.

Source code in trestle/core/commands/author/catalog.py

def generate_markdown(
    self,
    trestle_root: pathlib.Path,
    catalog_path: pathlib.Path,
    markdown_path: pathlib.Path,
    yaml_header: Dict[str, Any],
    overwrite_header_values: bool
) -> int:
    """Generate markdown for the controls in the catalog."""
    try:
        catalog = load_validate_model_path(trestle_root, catalog_path)
        context = ControlContext.generate(
            ContextPurpose.CATALOG,
            True,
            trestle_root,
            markdown_path,
            cli_yaml_header=yaml_header,
            overwrite_header_values=overwrite_header_values,
            set_parameters_flag=True
        )
        catalog_api = CatalogAPI(catalog=catalog, context=context)
        catalog_api.write_catalog_as_markdown()

    except TrestleNotFoundError as e:
        raise TrestleError(f'Catalog {catalog_path} not found for load: {e}')
    except Exception as e:
        raise TrestleError(f'Error generating markdown for controls in {catalog_path}: {e}')

    return CmdReturnCodes.SUCCESS.value

Functions

handler: python