From 83c05bb9c0a57470ba21e0a9ba44a00c613a82e5 Mon Sep 17 00:00:00 2001 From: Seed Date: Thu, 21 Dec 2023 17:29:43 +0000 Subject: [PATCH] logging + config file --- README.md | 2 +- config.toml | 4 + inventory.yml | 2 +- {templates => platform_templates}/eos.j2 | 0 {templates => platform_templates}/ios.j2 | 0 render_engine.log | 214 +++++++++++++++++++++++ render_engine.py | 171 ++++++++++++++---- rendered_templates/device_b.txt | 78 +++++++++ 8 files changed, 433 insertions(+), 38 deletions(-) create mode 100644 config.toml rename {templates => platform_templates}/eos.j2 (100%) rename {templates => platform_templates}/ios.j2 (100%) create mode 100644 render_engine.log create mode 100644 rendered_templates/device_b.txt diff --git a/README.md b/README.md index 326c7a6..3b275ef 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ source $HOME/gc_integration_templater/venv/bin/activate python --version which python pip install --proxy http://asblcinfpxy01:3128 --upgrade pip -pip install --proxy http://asblcinfpxy01:3128 pyyaml jinja2 +pip install --proxy http://asblcinfpxy01:3128 pyyaml jinja2 toml deactivate ``` diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..f319f6f --- /dev/null +++ b/config.toml @@ -0,0 +1,4 @@ +config_contexts_dir = './config_contexts' +platform_templates_dir = './platform_templates' +rendered_templates_dir = './rendered_templates' +inventory_file = './inventory.yml' \ No newline at end of file diff --git a/inventory.yml b/inventory.yml index a2a111d..fa1e582 100644 --- a/inventory.yml +++ b/inventory.yml @@ -4,7 +4,7 @@ # content of inventory.yml # # router1_customer1: # include variables from config_contexts/device/router1_customer1.yml -# platform: eos # include template from templates/eos.j2 +# platform: eos # include template from platform_templates/eos.j2 # config_contexts: # create folders by the name of the key and files by the name of the key value(s), # site: dub # include variables from config_contexts/site/dub.yml # region: europe diff --git a/templates/eos.j2 b/platform_templates/eos.j2 similarity index 100% rename from templates/eos.j2 rename to platform_templates/eos.j2 diff --git a/templates/ios.j2 b/platform_templates/ios.j2 similarity index 100% rename from templates/ios.j2 rename to platform_templates/ios.j2 diff --git a/render_engine.log b/render_engine.log new file mode 100644 index 0000000..d9fa6a8 --- /dev/null +++ b/render_engine.log @@ -0,0 +1,214 @@ +2023-12-12 11:35:22, ERROR | render_engine.py:304 | MainThread: device config_context render issue: device_a +clashing key found in equally weighted contexts: +key: harry_potter +contexts: ['switch', 'vpn'] + +clashing key found in equally weighted contexts: +key: gabbys_doll_house +contexts: ['router', 'switch'] + +2023-12-12 11:35:22, ERROR | render_engine.py:350 | MainThread: device template render issue: device_a +template: ./platform_templates/ios.j2 +empty config_context: {} +2023-12-12 11:35:22, INFO | render_engine.py:364 | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt +2023-12-12 11:35:22, INFO | render_engine.py:356 | MainThread: device_b rendered file content: + +**** start of template **** + +0) display the config_context: + + top level context item: + + {'best_unicorn': ['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'], 'cbeebies_schedule': '18/11/2023', 'device': 'device_b', 'gabbys_doll_house': 1, 'harry_potter': 1, 'octonaughts': 1, 'region': 'europe', 'site': 'dublin'} + + selecting the best unicorn with config_context['best_unicorn'][0]: + + sugar socks + + accessing a key pair injected into the jinja environment global variables: + (in this case the config_context printed out in human readable yaml format) + + best_unicorn: + - sugar socks + - star light + - sparkle + - twinkle + - glitter love + cbeebies_schedule: 18/11/2023 + device: device_b + gabbys_doll_house: 1 + harry_potter: 1 + octonaughts: 1 + region: europe + site: dublin + + + loop through the config_context variable in jinja: + + best_unicorn:['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'] + cbeebies_schedule:18/11/2023 + device:device_b + gabbys_doll_house:1 + harry_potter:1 + octonaughts:1 + region:europe + site:dublin + +1) source template ./device/device_b.yml + +2) source template ./device/region/europe.yml + +3) source template ./device/site/dublin.yml + +4) source template ./device/role/.yml + these contexts happens to be a list in the inventory + files under this directory happen to have the same weight(they dont have to) + thus these files do not have the same keys as they would clash + + gabbys_doll_house = 1 + + octonaughts = 1 + + notice best_unicorn is in the all.yml context and router.yml context + router.yml has a higher precedence(lower weight value) + + - sugar socks + - star light + - sparkle + - twinkle + - glitter love + + harry_potter = 1 + + notice cbeebies_schedule is in the all.yml context and vpn.yml context + all.yml context is weighted last/-1, any other context will overwrite it + vpn.yml has a higher precedence(lower weight value) + + cbeebies_schedule = 18/11/2023 + +4) source template ./device/role/all.yml + nothing is rendered using this context, all variables are overwritten by contexts with higher precedence + this would be a good place to put service accounts in ACLs or legal disclaimers + +**** end of template **** + +2023-12-12 11:36:41, ERROR | render_engine.py:304 | MainThread: device config_context render issue: device_a +clashing key found in equally weighted contexts: +key: gabbys_doll_house +contexts: ['router', 'switch'] + +clashing key found in equally weighted contexts: +key: harry_potter +contexts: ['switch', 'vpn'] + +2023-12-12 11:36:41, ERROR | render_engine.py:350 | MainThread: device template render issue: device_a +template: ./platform_templates/ios.j2 +empty config_context: {} +2023-12-12 11:36:41, INFO | render_engine.py:364 | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt +2023-12-12 11:36:41, INFO | render_engine.py:356 | MainThread: device_b rendered file content: + +**** start of template **** + +0) display the config_context: + + top level context item: + + {'best_unicorn': ['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'], 'cbeebies_schedule': '18/11/2023', 'device': 'device_b', 'gabbys_doll_house': 1, 'harry_potter': 1, 'octonaughts': 1, 'region': 'europe', 'site': 'dublin'} + + selecting the best unicorn with config_context['best_unicorn'][0]: + + sugar socks + + accessing a key pair injected into the jinja environment global variables: + (in this case the config_context printed out in human readable yaml format) + + best_unicorn: + - sugar socks + - star light + - sparkle + - twinkle + - glitter love + cbeebies_schedule: 18/11/2023 + device: device_b + gabbys_doll_house: 1 + harry_potter: 1 + octonaughts: 1 + region: europe + site: dublin + + + loop through the config_context variable in jinja: + + best_unicorn:['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'] + cbeebies_schedule:18/11/2023 + device:device_b + gabbys_doll_house:1 + harry_potter:1 + octonaughts:1 + region:europe + site:dublin + +1) source template ./device/device_b.yml + +2) source template ./device/region/europe.yml + +3) source template ./device/site/dublin.yml + +4) source template ./device/role/.yml + these contexts happens to be a list in the inventory + files under this directory happen to have the same weight(they dont have to) + thus these files do not have the same keys as they would clash + + gabbys_doll_house = 1 + + octonaughts = 1 + + notice best_unicorn is in the all.yml context and router.yml context + router.yml has a higher precedence(lower weight value) + + - sugar socks + - star light + - sparkle + - twinkle + - glitter love + + harry_potter = 1 + + notice cbeebies_schedule is in the all.yml context and vpn.yml context + all.yml context is weighted last/-1, any other context will overwrite it + vpn.yml has a higher precedence(lower weight value) + + cbeebies_schedule = 18/11/2023 + +4) source template ./device/role/all.yml + nothing is rendered using this context, all variables are overwritten by contexts with higher precedence + this would be a good place to put service accounts in ACLs or legal disclaimers + +**** end of template **** + +2023-12-12 11:36:55, ERROR | render_engine.py:303 | MainThread: device config_context render issue: device_a +clashing key found in equally weighted contexts: +key: gabbys_doll_house +contexts: ['router', 'switch'] + +clashing key found in equally weighted contexts: +key: harry_potter +contexts: ['switch', 'vpn'] + +2023-12-12 11:36:55, ERROR | render_engine.py:349 | MainThread: device template render issue: device_a +template: ./platform_templates/ios.j2 +empty config_context: {} +2023-12-12 11:36:55, INFO | render_engine.py:363 | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt +2023-12-12 11:46:39, ERROR | render_engine.py:291 | MainThread: device config_context render issue: device_a +clashing key found in equally weighted contexts: +key: harry_potter +contexts: ['switch', 'vpn'] + +clashing key found in equally weighted contexts: +key: gabbys_doll_house +contexts: ['router', 'switch'] + +2023-12-12 11:46:39, ERROR | render_engine.py:335 | MainThread: device template render issue: device_a +template: ./platform_templates/ios.j2 +empty config_context: {} +2023-12-12 11:46:39, INFO | render_engine.py:347 | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt diff --git a/render_engine.py b/render_engine.py index 34bb44d..9ccc1d7 100755 --- a/render_engine.py +++ b/render_engine.py @@ -5,13 +5,94 @@ from pathlib import Path import os import yaml import jinja2 +import toml +import logging -class FileChecks: +## global logger +logger = logging.getLogger('main') +logger.setLevel(logging.INFO) +console = logging.StreamHandler() +file = logging.FileHandler('render_engine.log') +logger.addHandler(console) +logger.addHandler(file) +formatter = logging.Formatter( + fmt="%(asctime)s, %(levelname)-8s | %(filename)-15s:%(lineno)-5s | %(threadName)-1s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S") +console.setFormatter(formatter) +file.setFormatter(formatter) + +## logic +class ReadConfig(): + 'read or generate script config file' + def __init__(self, config_file): + # dirty way to write a text block + self.example_config = """\ +config_contexts_dir = './config_contexts' +platform_templates_dir = './platform_templates' +rendered_templates_dir = './rendered_templates' +inventory_file = './inventory.yml'\ +""" + self.required_variables = ['config_contexts_dir', 'platform_templates_dir', 'rendered_templates_dir', 'inventory_file'] + self.config_file = config_file + self.config_dict = self.__find_config(self.example_config, self.config_file) + self.__parseConfig(self.config_dict) + + def __find_config(self, example_config, config_file): + if not Path(config_file).exists(): + with open(config_file,'w') as config: + config.write(example_config) + logger.error(f'config missing, wrote example to {config_file} please modify and re-run') + sys.exit(0) + else: + try: + config_dict = toml.load(config_file) + except: + logger.error(f'invalid TOML format: {config_file}') + sys.exit(1) + return config_dict + + def __parseConfig(self, config_dict): + # missing entries listed in self.required_variables + for i in self.required_variables: + try: + test = config_dict[i] + except: + logger.error(f'missing config entry: {i}') + sys.exit(1) + # empty entries in self.required_variables + for i in self.required_variables: + if not len(config_dict[i]): + logger.error(f'empty config entry: {i}') + sys.exit(1) + # missing files/directory in self.required_variables + missing = False + for i in self.required_variables: + if '_dir' in i: + directory = Path(config_dict[i]) + if not directory.is_dir(): + logger.error(f'missing directory: {i}= {config_dict[i]}') + missing = True + if '_file' in i: + file = Path(config_dict[i]) + if not file.is_file(): + print(f'missing file: {i} = {config_dict[i]}') + missing =True + if missing: + sys.exit(1) + + def get_config(self): + return self.config_dict + + def get_config_item(self, config_item): + if config_item in self.required_variables: + return self.config_dict[config_item] + +class InventoryFileChecks: 'file check rules' - def __init__(self, config_contexts_dir, templates_dir, inventory_file): - self.config_contexts_dir = config_contexts_dir - self.templates_dir = templates_dir - self.inventory_file = inventory_file + def __init__(self, config_dict): + self.config_contexts_dir = config_dict['config_contexts_dir'] + self.templates_dir = config_dict['platform_templates_dir'] + self.inventory_file = config_dict['inventory_file'] self.inventory_dict = self.__read_inventory_file(self.inventory_file) self.__check_context_files(self.config_contexts_dir) self.__check_platform_templates(self.inventory_dict, self.templates_dir) @@ -22,13 +103,13 @@ class FileChecks: return yaml.safe_load(file) except Exception as ex: if type(ex).__name__ == 'ScannerError': - print(f'Error {inventory_file}: is not valid YAML') + logger.error(f'invalid YAML: {inventory_file}') sys.exit(1) elif type(ex).__name__ == 'FileNotFoundError': - print(f'Error {inventory_file}: {ex.args[1]}') + logger.error(f'missing inventory file: {inventory_file}') sys.exit(1) else: - print(f'Error: {inventory_file}') + logger.error(f'unable to load inventory file: {inventory_file}') sys.exit(1) def __check_context_files(self, config_contexts_dir): @@ -39,11 +120,11 @@ class FileChecks: file_names = [context_files[i]['file'] for i in context_files.keys()] duplicate_files = list(set([i for i in file_names if file_names.count(i) > 1])) if len(duplicate_files) >0: - print('==== to maintain readability there no logic to handle duplicate file names under the config_contexts directory') + logger.error(f'to maintain readability there no logic to handle duplicate file names under the config_contexts directory') for i in duplicate_files: for j in context_files.keys(): if context_files[j]['file'] == i: - print(f"duplicate file name {context_files[j]['file']} @ {j}") + logger.error(f"duplicate file name {context_files[j]['file']} @ {j}") sys.exit(1) def __check_platform_templates(self, inventory_dict, templates_dir): @@ -53,14 +134,16 @@ class FileChecks: if not template.is_file(): target_device_list = [k for k, v in inventory_dict.items() if v['platform'] == i] target_devices = '\n'.join(target_device_list) - print(f'==== missing template {template}, nothing will be rendered for devices:\n{target_devices}') + logger.error(f'missing template {template}, nothing will be rendered for devices:\n{target_devices}') -class LoadInventory(FileChecks): +class LoadInventory(InventoryFileChecks): 'load inventory' - def __init__(self, config_contexts_dir, templates_dir, inventory_file): - FileChecks.__init__(self, config_contexts_dir, templates_dir, inventory_file) + def __init__(self, config_dict): + self.config_dict = config_dict + InventoryFileChecks.__init__(self, self.config_dict) self.all_config_contexts, self.devices = self.__load_contexts(self.inventory_dict) - #print(yaml.dump(self.all_config_contexts)) + ## debug + # print(yaml.dump(self.all_config_contexts)) def __remove_meta(self, data): try: @@ -83,6 +166,7 @@ class LoadInventory(FileChecks): contexts_path.add(f'{self.config_contexts_dir}/{k}/{i}.yml') else: contexts_path.add(f'{self.config_contexts_dir}/{k}/{v}.yml') + ## debug # print(devices_path) # print(contexts_path) @@ -99,6 +183,7 @@ class LoadInventory(FileChecks): all_config_contexts.update({context: {'weight': data['_metadata']['weight'], 'content': self.__remove_meta(data)}}) except: pass + ## debug # print(all_config_contexts) # load device contexts @@ -152,12 +237,16 @@ class LoadInventory(FileChecks): device_contexts.update({v: self.all_config_contexts[v]}) return device_contexts + def get_rendered_templates_path(self): + return self.config_dict['rendered_templates_dir'] + class BuildDeviceContext: 'build config_context for a device' def __init__(self, LoadInventory_instance, device): self.device = device self.contexts = LoadInventory_instance.get_device_contexts(self.device) self.platform = LoadInventory_instance.get_device_platform(self.device) + ## debug # print(self.contexts) # print(self.platform) self.config_context = self.__config_context(self.device, self.contexts) @@ -172,6 +261,7 @@ class BuildDeviceContext: for k in contexts.keys(): if contexts[k]['weight'] == i: equal_weight[i].append(k) + ## debug # print(f'equal weight contexts: {equal_weight}') # list all keys in equally weighted contexts @@ -181,6 +271,7 @@ class BuildDeviceContext: for k in contexts[i]['content'].keys(): keys_list.append(k) keys_list = list(set(keys_list)) + ## debug # print(f'all keys in equal weight contexts: {keys_list}') # check each equally weighted context for clashing keys @@ -194,10 +285,10 @@ class BuildDeviceContext: if len(key_occurance) >1: equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}') if len(equal_weight_key_clash_errors) >0: - error_log = f'==== device config_context render issue: {device}\n' + error_log = f'device config_context render issue: {device}' for i in equal_weight_key_clash_errors: error_log = error_log + f'\n{i}\n' - print(error_log) + logger.error(error_log) return {} # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting top level keys where they exist (the last context being the lowest weight and highest precedence) @@ -206,6 +297,7 @@ class BuildDeviceContext: for k, v in contexts.items(): config_context = {**config_context, **v['content']} config_context = dict(sorted(config_context.items())) + ## debug # print(yaml.dump(config_context)) return config_context @@ -220,13 +312,13 @@ class RenderDeviceConfig(BuildDeviceContext): def __init__(self, LoadInventory_instance, device): BuildDeviceContext.__init__(self, LoadInventory_instance, device) self.device_template = LoadInventory_instance.get_device_template(self.device) + self.rendered_template_path = LoadInventory_instance.get_rendered_templates_path() self.rendered_template = self.__rendered_template(self.device, self.config_context, self.device_template) def __rendered_template(self, device, config_context, device_template): if len(config_context) >0: template_directory = os.path.dirname(device_template) template_file = os.path.basename(device_template) - # rendered_template_file = f'{device}.txt' environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_directory)) environment.trim_blocks = True environment.lstrip_blocks = True @@ -240,51 +332,58 @@ class RenderDeviceConfig(BuildDeviceContext): # rendered_template_content = template.render(config_context) return rendered_template_content else: - print(f'==== device template render issue: {device}\n\n{os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context {config_context} (clashing top level keys in config_contexts)\n') + logger.error(f'device template render issue: {device}\ntemplate: {os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context: {config_context}') return def print_rendered_template(self): if self.rendered_template is not None: - print(f'==== {self.device} file content:\n\n{self.rendered_template}\n') + logger.info(f'{self.device} rendered file content:\n\n{self.rendered_template}\n') def write_rendered_template(self): if self.rendered_template is not None: - # this needs more logic to pass another parameter for dest dir - rendered_template_file = f'{self.device}.txt' + rendered_template_file = f'{self.rendered_template_path}/{self.device}.txt' with open(rendered_template_file, mode="w", encoding="utf-8") as message: message.write(self.rendered_template) - print(f'==== {self.device} file written: {rendered_template_file}') + logger.info(f'{self.device} rendered file written: {rendered_template_file}') +## start def main(): - config_contexts_dir = './config_contexts' - templates_dir = './templates' - inventory_file = "./inventory.yml" - inventory = LoadInventory(config_contexts_dir, templates_dir, inventory_file) + + # config validation + config_file = 'config.toml' + config_obj = ReadConfig(config_file) + + # load inventory + inventory = LoadInventory(config_obj.get_config()) + ## debug # print(inventory.get_devices()) # print(yaml.dump(inventory.get_contexts())) + # render templates config_contexts_list = [] for i in inventory.get_devices(): context_obj = RenderDeviceConfig(inventory, i) if len(context_obj.get_config_context()) >0: + ## debug # print(f'device:\n{context_obj.get_device_name()}\n') # print(f'config_context:\n{yaml.dump(context_obj.get_config_context())}') config_contexts_list.append(context_obj) else: del context_obj + # write rendered configs for i in config_contexts_list: - i.print_rendered_template() i.write_rendered_template() + ## debug + # i.print_rendered_template() if __name__ == "__main__": main() -# turn this into a tool? -# - wrap up into parameterized script, try with a real template -# add comment block to each renderd config such as rendered date (maybe list templates in use?) -# needs a config file to point to contexts/folders, use toml top allow for comments -# - feedback, the templates directory is confusing, templates/platforms -# needs a rendered artefact directory parameter somewhere, maybe this should be a write_rendered_template parameter to allow it to be placed in another git repo -# needs proper logging -# needs a mode to create example files/dirs + +# parameters for: +# - non standard config file location +# - show me all devices in inventory +# - target specific device for view or render +# - view/render +# want a comment block or maybe for now just add a date/time in the file name - do this for now - this is going to be a git commit thing is there any point? diff --git a/rendered_templates/device_b.txt b/rendered_templates/device_b.txt new file mode 100644 index 0000000..dd47120 --- /dev/null +++ b/rendered_templates/device_b.txt @@ -0,0 +1,78 @@ +**** start of template **** + +0) display the config_context: + + top level context item: + + {'best_unicorn': ['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'], 'cbeebies_schedule': '18/11/2023', 'device': 'device_b', 'gabbys_doll_house': 1, 'harry_potter': 1, 'octonaughts': 1, 'region': 'europe', 'site': 'dublin'} + + selecting the best unicorn with config_context['best_unicorn'][0]: + + sugar socks + + accessing a key pair injected into the jinja environment global variables: + (in this case the config_context printed out in human readable yaml format) + + best_unicorn: + - sugar socks + - star light + - sparkle + - twinkle + - glitter love + cbeebies_schedule: 18/11/2023 + device: device_b + gabbys_doll_house: 1 + harry_potter: 1 + octonaughts: 1 + region: europe + site: dublin + + + loop through the config_context variable in jinja: + + best_unicorn:['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'] + cbeebies_schedule:18/11/2023 + device:device_b + gabbys_doll_house:1 + harry_potter:1 + octonaughts:1 + region:europe + site:dublin + +1) source template ./device/device_b.yml + +2) source template ./device/region/europe.yml + +3) source template ./device/site/dublin.yml + +4) source template ./device/role/.yml + these contexts happens to be a list in the inventory + files under this directory happen to have the same weight(they dont have to) + thus these files do not have the same keys as they would clash + + gabbys_doll_house = 1 + + octonaughts = 1 + + notice best_unicorn is in the all.yml context and router.yml context + router.yml has a higher precedence(lower weight value) + + - sugar socks + - star light + - sparkle + - twinkle + - glitter love + + harry_potter = 1 + + notice cbeebies_schedule is in the all.yml context and vpn.yml context + all.yml context is weighted last/-1, any other context will overwrite it + vpn.yml has a higher precedence(lower weight value) + + cbeebies_schedule = 18/11/2023 + +4) source template ./device/role/all.yml + nothing is rendered using this context, all variables are overwritten by contexts with higher precedence + this would be a good place to put service accounts in ACLs or legal disclaimers + +**** end of template **** \ No newline at end of file