36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437 | class Folders(AuthorCommonCommand):
"""Markdown governed folders - enforcing consistent files and templates across directories."""
name = 'folders'
def _init_arguments(self) -> None:
self.add_argument(
author_const.GH_SHORT, author_const.GH_LONG, help=author_const.GH_HELP, default=None, type=str
)
self.add_argument(
author_const.SHORT_HEADER_VALIDATE,
author_const.LONG_HEADER_VALIDATE,
help=author_const.HEADER_VALIDATE_HELP,
action='store_true'
)
self.add_argument(
author_const.HOV_SHORT, author_const.HOV_LONG, help=author_const.HOV_HELP, action='store_true'
)
self.add_argument(
author_const.SHORT_TEMPLATE_VERSION,
author_const.LONG_TEMPLATE_VERSION,
help=author_const.TEMPLATE_VERSION_HELP,
action='store'
)
self.add_argument(
author_const.SHORT_IGNORE, author_const.LONG_IGNORE, help=author_const.IGNORE_HELP, default=None, type=str
)
self.add_argument(author_const.MODE_ARG_NAME, choices=author_const.MODE_CHOICES)
tn_help_str = '\n'.join(
[
'The name of the the task to be governed.',
'',
'The template files are at .trestle/author/[task-name],',
'where the directory tree established and the markdown files within that directory'
+ 'tree are enforced.'
]
)
self.add_argument(
author_const.TASK_NAME_SHORT, author_const.TASK_NAME_LONG, help=tn_help_str, required=True, type=str
)
self.add_argument(
author_const.SHORT_README_VALIDATE,
author_const.LONG_README_VALIDATE,
help=author_const.README_VALIDATE_FOLDERS_HELP,
action='store_true'
)
self.add_argument(
author_const.TEMPLATE_TYPE_VALIDATE_SHORT,
author_const.TEMPLATE_TYPE_VALIDATE_LONG,
help=author_const.TEMPLATE_TYPE_VALIDATE_HELP,
action='store_true'
)
def _run(self, args: argparse.Namespace) -> int:
try:
if self._initialize(args):
raise TrestleError(f'Error when initializing trestle folders command with args: {args}')
if args.mode == 'create-sample':
status = self.create_sample()
elif args.mode == 'template-validate':
status = self.template_validate(
args.header_validate, args.header_only_validate, args.governed_heading, args.readme_validate
)
elif args.mode == 'setup':
status = self.setup_template(args.template_version)
elif args.mode == 'validate':
# mode is validate
status = self.validate(
args.header_validate,
args.header_only_validate,
args.governed_heading,
args.readme_validate,
args.template_version,
args.ignore,
args.validate_template_type
)
else:
raise TrestleIncorrectArgsError(f'Unsupported mode: {args.mode} for folders command.')
return status
except Exception as e: # pragma: no cover
return handle_generic_command_exception(e, logger, 'Error occurred when running trestle author folders')
def setup_template(self, template_version: str) -> int:
"""Create structure to allow markdown template enforcement."""
if not self.task_path.exists():
self.task_path.mkdir(exist_ok=True, parents=True)
elif self.task_path.is_file():
raise TrestleError(f'Task path: {self.rel_dir(self.task_path)} is a file not a directory.')
if not self.template_dir.exists():
self.template_dir.mkdir(exist_ok=True, parents=True)
elif self.template_dir.is_file():
raise TrestleError(f'Template path: {self.rel_dir(self.template_dir)} is a file not a directory.')
template_file_a_md = self.template_dir / 'a_template.md'
template_file_another_md = self.template_dir / 'another_template.md'
template_file_drawio = self.template_dir / 'architecture.drawio'
TemplateVersioning.write_versioned_template(
'template.md', self.template_dir, template_file_a_md, template_version
)
TemplateVersioning.write_versioned_template(
'template.md', self.template_dir, template_file_another_md, template_version
)
TemplateVersioning.write_versioned_template(
'template.drawio', self.template_dir, template_file_drawio, template_version
)
return CmdReturnCodes.SUCCESS.value
def template_validate(
self, validate_header: bool, validate_only_header: bool, heading: str, readme_validate: bool
) -> int:
"""Validate that the template is acceptable markdown."""
if not self.template_dir.is_dir():
raise TrestleError(
f'Template directory {self.rel_dir(self.template_dir)} for task {self.task_name} does not exist.'
)
# get list of files:
template_files = self.template_dir.rglob('*')
for template_file in template_files:
try:
if not file_utils.is_local_and_visible(template_file):
continue
elif template_file.is_dir():
continue
elif template_file.suffix.lower() == const.MARKDOWN_FILE_EXT:
if not readme_validate and template_file.name == 'readme.md':
raise TrestleError('Template directory contains a readme.md file and readme validation is off.')
md_api = MarkdownAPI()
md_api.load_validator_with_template(
template_file, validate_header, not validate_only_header, heading
)
elif template_file.suffix.lower().lstrip('.') == 'drawio':
_ = draw_io.DrawIOMetadataValidator(template_file)
else:
logger.info(
f'File: {self.rel_dir(template_file)} within the template directory was ignored'
+ ' as it is not markdown.'
)
except Exception as ex:
raise TrestleError(
f'Template file {self.rel_dir(template_file)} for task {self.task_name}'
+ f' failed to validate due to {ex}'
)
logger.info(f'TEMPLATES VALID: {self.task_name}.')
return CmdReturnCodes.SUCCESS.value
def _measure_template_folder(
self,
instance_dir: pathlib.Path,
validate_header: bool,
validate_only_header: bool,
governed_heading: str,
readme_validate: bool,
template_version: str,
ignore: str,
validate_by_type_field: bool,
) -> bool:
"""
Validate instances against templates.
Validation will succeed if:
1. All template files from the specified version are present in the task
2. All of the instances are valid
"""
all_versioned_templates: Dict[str, Any] = {}
instance_version = template_version
instance_file_names: List[pathlib.Path] = []
# Fetch all instances versions and build dictionary of required template files
for instance_file in instance_dir.iterdir():
if not file_utils.is_local_and_visible(instance_file):
continue
if not instance_file.is_file():
continue
if instance_file.name.lower() == 'readme.md' and not readme_validate:
continue
if ignore:
p = re.compile(ignore)
matched = p.match(instance_file.parts[-1])
if matched is not None:
logger.info(f'Ignoring file {instance_file} from validation.')
continue
instance_file_name = instance_file.relative_to(instance_dir)
instance_file_names.append(instance_file_name)
if instance_file.suffix == const.MARKDOWN_FILE_EXT:
md_api = MarkdownAPI()
versioned_template_dir = None
# checks on naming template name out of type header if needed
if validate_by_type_field:
template_name = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_TYPE_HEADER
)
if template_name is None:
logger.warning(
f'INVALID: Instance file {instance_file_name} does not have'
f' {author_const.TEMPLATE_TYPE_HEADER}'
' field in its header and can not be validated using optional parameter validate'
' template type field'
)
return False
template_name = template_name + '.md'
else:
template_name = instance_file_name
if template_version != '':
template_file = self.template_dir / template_name
versioned_template_dir = self.template_dir
else:
instance_version = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_VERSION_HEADER
)
if instance_version is None:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version
)
template_file = versioned_template_dir / template_name
# Check if instance is in the available templates,
# additional files are allowed but should not be validated.
templates = self._get_templates(versioned_template_dir, readme_validate)
is_template_present = False
template_type_is_valid = False
for template in templates:
# checks if valdation needs to check on x-trestle-template-type field on header
if validate_by_type_field:
instance_template_type = md_api.processor.fetch_value_from_header(
instance_file, author_const.TEMPLATE_TYPE_HEADER
)
if template.stem == instance_template_type:
is_template_present = True
template_type_is_valid = True
break
# validation through template type field is not needed and performs validation
# through file name flow as usual
else:
if template.name == str(instance_file_name):
is_template_present = True
break
if not is_template_present:
logger.info(
f'INFO: File{instance_file} will not be validated '
f'as its name does not match any template file.'
)
continue
if instance_version not in all_versioned_templates.keys():
all_versioned_templates[instance_version] = dict.fromkeys(
[t.relative_to(versioned_template_dir) for t in templates], False
)
if instance_file_name in all_versioned_templates[instance_version] or template_type_is_valid:
# validate
md_api.load_validator_with_template(
template_file, validate_header, not validate_only_header, governed_heading
)
status = md_api.validate_instance(instance_file)
if not status:
logger.warning(
f'INVALID: Markdown file {instance_file} failed validation against' + f' {template_file}'
)
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
if template_type_is_valid:
template_file_name = [
temp.relative_to(versioned_template_dir) for temp in templates if temp.name == template_name
]
all_versioned_templates[instance_version][template_file_name[0]] = True
else:
all_versioned_templates[instance_version][instance_file_name] = True
elif instance_file.suffix == const.DRAWIO_FILE_EXT:
drawio = draw_io.DrawIO(instance_file)
metadata = drawio.get_metadata()[0]
versioned_template_dir = None
if template_version != '':
template_file = self.template_dir / instance_file_name
versioned_template_dir = self.template_dir
else:
if author_const.TEMPLATE_VERSION_HEADER in metadata.keys():
instance_version = metadata[author_const.TEMPLATE_VERSION_HEADER]
else:
instance_version = '0.0.1' # backward compatibility
versioned_template_dir = TemplateVersioning.get_versioned_template_dir(
self.template_dir, instance_version
)
template_file = versioned_template_dir / instance_file_name
if instance_version not in all_versioned_templates.keys():
templates = self._get_templates(versioned_template_dir, readme_validate)
all_versioned_templates[instance_version] = dict.fromkeys(
[t.relative_to(versioned_template_dir) for t in templates], False
)
if instance_file_name in all_versioned_templates[instance_version]:
# validate
drawio_validator = draw_io.DrawIOMetadataValidator(template_file)
status = drawio_validator.validate(instance_file)
if not status:
logger.warning(
f'INVALID: Drawio file {instance_file} failed validation against' + f' {template_file}'
)
return False
else:
logger.info(f'VALID: {instance_file}')
# mark template as present
all_versioned_templates[instance_version][instance_file_name] = True
else:
logger.debug(f'Unsupported extension of the instance file: {instance_file}, will not be validated.')
# Check that all template files are present
for version in all_versioned_templates.keys():
for template in all_versioned_templates[version]:
if not all_versioned_templates[version][template]:
logger.warning(
f'Required template file {template} does not exist in measured instance' + f'{instance_dir}'
)
return False
return True
def _get_templates(self, versioned_template_dir: pathlib.Path, readme_validate: bool) -> List[pathlib.Path]:
"""Get templates for the given version."""
templates = list(
filter(
lambda p: file_utils.is_local_and_visible(p) and p.is_file()
and # noqa: W504 - conflicting lint and formatting
(p.suffix == const.MARKDOWN_FILE_EXT or p.suffix == const.DRAWIO_FILE_EXT),
versioned_template_dir.iterdir()
)
)
if not readme_validate:
templates = list(filter(lambda p: p.name.lower() != 'readme.md', templates))
return templates
def create_sample(self) -> int:
"""
Create a sample folder within the task and populate with template content.
Returns:
Unix return code for running sample as a command.
"""
ii = 0
while True:
sample_path = self.task_path / f'sample_folder_{ii}'
if sample_path.exists():
ii = ii + 1
continue
shutil.copytree(str(self.template_dir), str(sample_path))
return CmdReturnCodes.SUCCESS.value
def validate(
self,
validate_header: bool,
validate_only_header: bool,
governed_heading: str,
readme_validate: bool,
template_version: str,
ignore: str,
validate_by_type_field: bool,
) -> int:
"""Validate task."""
if not self.task_path.is_dir():
raise TrestleError(f'Task directory {self.task_path} does not exist. Exiting validate.')
for task_instance in self.task_path.iterdir():
if task_instance.is_dir():
if file_utils.is_symlink(task_instance):
continue
result = self._measure_template_folder(
task_instance,
validate_header,
validate_only_header,
governed_heading,
readme_validate,
template_version,
ignore,
validate_by_type_field
)
if not result:
raise TrestleError(
'Governed-folder validation failed for task'
+ f'{self.task_name} on directory {self.rel_dir(task_instance)}'
)
else:
logger.info(
f'Unexpected file {self.rel_dir(task_instance)} identified in {self.task_name}'
+ ' directory, ignoring.'
)
return CmdReturnCodes.SUCCESS.value
|