commit 74f434d971d9fdcaa4d7c826aedebc5957f5bbe7 Author: tseed Date: Mon Nov 20 12:34:46 2023 +0000 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..27c2358 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +venv_nautofake/ diff --git a/config_contexts/all.yml b/config_contexts/all.yml new file mode 100644 index 0000000..171290a --- /dev/null +++ b/config_contexts/all.yml @@ -0,0 +1,11 @@ +--- +_metadata: + name: "all" + description: "Global Definitions" + is_active: true +cbeebies_schedule: "16/11/2023" +best_unicorn: + - sparkle + - "star light" + - "glitter love" + - twinkle \ No newline at end of file diff --git a/config_contexts/device/device_a.yml b/config_contexts/device/device_a.yml new file mode 100644 index 0000000..ba09bdd --- /dev/null +++ b/config_contexts/device/device_a.yml @@ -0,0 +1 @@ +device: device_a \ No newline at end of file diff --git a/config_contexts/device/device_b.yml b/config_contexts/device/device_b.yml new file mode 100644 index 0000000..440da07 --- /dev/null +++ b/config_contexts/device/device_b.yml @@ -0,0 +1 @@ +device: device_b \ No newline at end of file diff --git a/config_contexts/region/asia.yml b/config_contexts/region/asia.yml new file mode 100644 index 0000000..b4c0ab8 --- /dev/null +++ b/config_contexts/region/asia.yml @@ -0,0 +1,6 @@ +_metadata: + name: "all" + weight: 200 + description: "Region Asia Definitions" + is_active: true +region: asia \ No newline at end of file diff --git a/config_contexts/region/europe.yml b/config_contexts/region/europe.yml new file mode 100644 index 0000000..f629865 --- /dev/null +++ b/config_contexts/region/europe.yml @@ -0,0 +1,6 @@ +_metadata: + name: "all" + weight: 200 + description: "Region Europe Definitions" + is_active: true +region: europe \ No newline at end of file diff --git a/config_contexts/region/north_america.yml b/config_contexts/region/north_america.yml new file mode 100644 index 0000000..dabf0e9 --- /dev/null +++ b/config_contexts/region/north_america.yml @@ -0,0 +1,6 @@ +_metadata: + name: "all" + weight: 200 + description: "Region North America Definitions" + is_active: true +region: north_america \ No newline at end of file diff --git a/config_contexts/region/south_america.yml b/config_contexts/region/south_america.yml new file mode 100644 index 0000000..58bc252 --- /dev/null +++ b/config_contexts/region/south_america.yml @@ -0,0 +1,6 @@ +_metadata: + name: "all" + weight: 200 + description: "Region South America Definitions" + is_active: true +region: south_america \ No newline at end of file diff --git a/config_contexts/role/router.yml b/config_contexts/role/router.yml new file mode 100644 index 0000000..44f5d47 --- /dev/null +++ b/config_contexts/role/router.yml @@ -0,0 +1,13 @@ +_metadata: + name: "all" + weight: 400 + description: "Router Definitions" + is_active: true +octonaughts: 1 +gabbys_doll_house: 1 +best_unicorn: + - "sugar socks" + - "star light" + - sparkle + - twinkle + - "glitter love" diff --git a/config_contexts/role/switch.yml b/config_contexts/role/switch.yml new file mode 100644 index 0000000..1a8538a --- /dev/null +++ b/config_contexts/role/switch.yml @@ -0,0 +1,7 @@ +_metadata: + name: "all" + weight: 400 + description: "Switch Definitions" + is_active: true +harry_potter: 1 +gabbys_doll_house: 1 \ No newline at end of file diff --git a/config_contexts/role/vpn.yml b/config_contexts/role/vpn.yml new file mode 100644 index 0000000..51993f7 --- /dev/null +++ b/config_contexts/role/vpn.yml @@ -0,0 +1,7 @@ +_metadata: + name: "all" + weight: 400 + description: "VPN Definitions" + is_active: true +harry_potter: 1 +cbeebies_schedule: "17/11/2023" \ No newline at end of file diff --git a/config_contexts/site/dub.yml b/config_contexts/site/dub.yml new file mode 100644 index 0000000..4113c31 --- /dev/null +++ b/config_contexts/site/dub.yml @@ -0,0 +1,7 @@ +_metadata: + name: "all" + weight: 300 + description: "Site Dublin Definitions" + is_active: true +site: dublin +cbeebies_schedule: "18/11/2023" \ No newline at end of file diff --git a/config_contexts/site/lon.yml b/config_contexts/site/lon.yml new file mode 100644 index 0000000..0d3ac0b --- /dev/null +++ b/config_contexts/site/lon.yml @@ -0,0 +1,6 @@ +_metadata: + name: "all" + weight: 300 + description: "Site London Definitions" + is_active: true +site: london diff --git a/inventory.yml b/inventory.yml new file mode 100644 index 0000000..a2a111d --- /dev/null +++ b/inventory.yml @@ -0,0 +1,59 @@ +# the inventory file dictates the content of the config_context dictionary item used to render jinja2 templates +# this could be generated by a csv, populated in a database or simply exist as a flat file + +# content of inventory.yml +# +# router1_customer1: # include variables from config_contexts/device/router1_customer1.yml +# platform: eos # include template from templates/eos.j2 +# config_contexts: # create folders by the name of the key and files by the name of the key value(s), +# site: dub # include variables from config_contexts/site/dub.yml +# region: europe +# role: # include variables from config_contexts/role/router.yml and config_contexts/role/vpn.yml, list may contain any number of items +# - router +# - vpn + +# content of config_contexts/device/router1_customer1.yml +# there are no special/functional items +# the device context has weight 0, top precedence +# keys in this file will overwrite any duplicate keys imported from any other context. +# +# key_name: "some_value" + +# content of files under the context_contexts folder, example config_contexts/site/dub.yml +# the _metadata object, keys 'weight'(int) and is_active(bool) are functional, they control the order of precedence in which the keys are merged to the config_context +# +# _metadata: +# name: "dublin" +# weight: 300 +# description: "Site Dublin Definitions" +# is_active: true +# site: dublin +# cbeebies_schedule: "18/11/2023" + +device_a: + platform: ios + config_contexts: + region: europe + site: lon + role: + - router + - switch + - vpn + - firewall +device_b: + platform: eos + config_contexts: + site: dub + region: europe + role: + - router + - vpn +# device_c: +# config_contexts: +# region: asia +# platform: ios +# device_d: +# config_contexts: +# region: south_america + + diff --git a/python_venv.md b/python_venv.md new file mode 100644 index 0000000..8574ff0 --- /dev/null +++ b/python_venv.md @@ -0,0 +1,13 @@ +# Python venv + +- Host with Python version 3.6+ + +```sh +python3 -m venv $HOME/nautofake/venv_nautofake +source $HOME/nautofake/venv_nautofake/bin/activate +python --version +which python +pip install --upgrade pip +pip install pyyaml jinja2 +deactivate +''' \ No newline at end of file diff --git a/render_engine.py b/render_engine.py new file mode 100644 index 0000000..f471c85 --- /dev/null +++ b/render_engine.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python + +import sys +from pathlib import Path +import os +import yaml +import jinja2 + +class FileChecks: + 'file check rules' + def __init__(self, config_contexts_dir, templates_dir, inventory_file): + self.config_contexts_dir = config_contexts_dir + self.templates_dir = templates_dir + self.inventory_file = inventory_file + self.inventory_dict = self.__read_inventory_file(self.inventory_file) + self.__check_context_files(self.config_contexts_dir) + self.__check_platform_templates(self.inventory_dict, self.templates_dir) + + def __read_inventory_file(self, inventory_file): + try: + with open(inventory_file, "r") as file: + return yaml.safe_load(file) + except Exception as ex: + if type(ex).__name__ == 'ScannerError': + print(f'Error {inventory_file}: is not valid YAML') + sys.exit(1) + elif type(ex).__name__ == 'FileNotFoundError': + print(f'Error {inventory_file}: {ex.args[1]}') + sys.exit(1) + else: + print(f'Error: {inventory_file}') + sys.exit(1) + + def __check_context_files(self, config_contexts_dir): + context_files = {} + # check for contexts directory - need an all file, dont need a device file + for path in Path(config_contexts_dir).rglob('*.yml'): + context_files.update({str(path): { 'file': path.name,'path': str(path.parent)}}) + file_names = [context_files[i]['file'] for i in context_files.keys()] + duplicate_files = list(set([i for i in file_names if file_names.count(i) > 1])) + if len(duplicate_files) >0: + print('==== to maintain readability there no logic to handle duplicate file names under the config_contexts directory') + for i in duplicate_files: + for j in context_files.keys(): + if context_files[j]['file'] == i: + print(f"duplicate file name {context_files[j]['file']} @ {j}") + sys.exit(1) + + def __check_platform_templates(self, inventory_dict, templates_dir): + platforms = set([inventory_dict[i]['platform'] for i in inventory_dict.keys()]) + for i in platforms: + template = Path(f'{templates_dir}/{i}.j2') + if not template.is_file(): + target_device_list = [k for k, v in inventory_dict.items() if v['platform'] == i] + target_devices = '\n'.join(target_device_list) + print(f'==== missing template {template}, nothing will be rendered for devices:\n{target_devices}') + +class LoadInventory(FileChecks): + 'load inventory' + def __init__(self, config_contexts_dir, templates_dir, inventory_file): + FileChecks.__init__(self, config_contexts_dir, templates_dir, inventory_file) + self.all_config_contexts, self.devices = self.__load_contexts(self.inventory_dict) + #print(yaml.dump(self.all_config_contexts)) + + def __remove_meta(self, data): + try: + del data['_metadata'] + except KeyError: + pass + return data + + def __load_contexts(self, inventory_dict): + # get path of all device and context files + devices_path = set() + contexts_path = set() + devices = [] + for i in inventory_dict.keys(): + devices_path.add(f'{self.config_contexts_dir}/device/{i}.yml') + devices.append(i) + for k, v in inventory_dict[i]['config_contexts'].items(): + if isinstance(v, list): + for i in v: + contexts_path.add(f'{self.config_contexts_dir}/{k}/{i}.yml') + else: + contexts_path.add(f'{self.config_contexts_dir}/{k}/{v}.yml') + # print(devices_path) + # print(contexts_path) + + # load config contexts + all_config_contexts = {} + for i in contexts_path: + try: + with open(i, 'r') as file: + data = yaml.safe_load(file) + if data['_metadata']['weight'] and data['_metadata']['is_active']: + # build the context_name from the file parent directory + # context naming could be sourced from the metadata dict but the potential for duplicate filenames under config_contexts subdirectories is to be discouraged + context = os.path.splitext(os.path.basename(i))[0] + all_config_contexts.update({context: {'weight': data['_metadata']['weight'], 'content': self.__remove_meta(data)}}) + except: + pass + # print(all_config_contexts) + + # load device contexts + for i in devices_path: + try: + with open(i, 'r') as file: + data = yaml.safe_load(file) + context = os.path.splitext(os.path.basename(i))[0] + all_config_contexts.update({context: {'weight': 0, 'content': self.__remove_meta(data)}}) + except: + pass + + # load all contexts + all_weight = sorted([all_config_contexts[k]['weight'] for k, v in all_config_contexts.items()], reverse=True) + max_weight = all_weight[0]+1 + try: + with open(f'{self.config_contexts_dir}/all.yml', 'r') as file: + data = yaml.safe_load(file) + if data['_metadata']['is_active']: + all_config_contexts.update({'all': {'weight': max_weight, 'content': self.__remove_meta(data)}}) + except: + pass + + return all_config_contexts, devices + + def get_contexts(self): + return self.all_config_contexts + + def get_devices(self): + return self.devices + + def get_device_platform(self, device): + return self.inventory_dict[device]['platform'] + + def get_device_template(self, device): + platform = self.get_device_platform(device) + return f'{self.templates_dir}/{platform}.j2' + + def get_device_contexts(self, device): + device_contexts = {} + all_device_contexts = ['all', device] + for i in all_device_contexts: + if i in self.all_config_contexts: + device_contexts.update({i: self.all_config_contexts[i]}) + for k, v in self.inventory_dict[device]['config_contexts'].items(): + if isinstance(v, list): + for i in v: + if i in self.all_config_contexts: + device_contexts.update({i: self.all_config_contexts[i]}) + elif v in self.all_config_contexts: + device_contexts.update({v: self.all_config_contexts[v]}) + return device_contexts + +class BuildDeviceContext: + 'build config_context for a device' + def __init__(self, LoadInventory_instance, device): + self.device = device + self.contexts = LoadInventory_instance.get_device_contexts(self.device) + self.platform = LoadInventory_instance.get_device_platform(self.device) + # print(self.contexts) + # print(self.platform) + self.config_context = self.__config_context(self.device, self.contexts) + + def __config_context(self, device, contexts): + # check for equally weighted clashing keys + all_weight = sorted([contexts[k]['weight'] for k, v in contexts.items()], reverse=True) + dupe_weight = list(set([x for x in all_weight if all_weight.count(x) > 1])) + equal_weight = {} + for i in dupe_weight: + equal_weight.update({i: []}) + for k in contexts.keys(): + if contexts[k]['weight'] == i: + equal_weight[i].append(k) + # print(f'equal weight contexts: {equal_weight}') + + # list all keys in equally weighted contexts + keys_list = [] + for k in equal_weight.keys(): + for i in equal_weight[k]: + for k in contexts[i]['content'].keys(): + keys_list.append(k) + keys_list = list(set(keys_list)) + # print(f'all keys in equal weight contexts: {keys_list}') + + # check each equally weighted context for clashing keys + equal_weight_key_clash_errors = [] + for w in equal_weight: + for k in keys_list: + key_occurance = [] + for i in equal_weight[w]: + if k in contexts[i]['content']: + key_occurance.append(i) + if len(key_occurance) >1: + equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}') + if len(equal_weight_key_clash_errors) >0: + error_log = f'==== device config_context render issue: {device}\n' + for i in equal_weight_key_clash_errors: + error_log = error_log + f'\n{i}\n' + print(error_log) + return {} + + # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting top level keys where they exist (the last context being the lowest weight and highest precedence) + config_context = {} + contexts = {k: v for k, v in sorted(contexts.items(), key=lambda x: x[1]['weight'], reverse=True)} + for k, v in contexts.items(): + config_context = {**config_context, **v['content']} + config_context = dict(sorted(config_context.items())) + # print(yaml.dump(config_context)) + return config_context + + def get_config_context(self): + return self.config_context + + def get_device_name(self): + return self.device + +class RenderDeviceConfig(BuildDeviceContext): + 'build config_context for a device' + def __init__(self, LoadInventory_instance, device): + BuildDeviceContext.__init__(self, LoadInventory_instance, device) + self.device_template = LoadInventory_instance.get_device_template(self.device) + self.rendered_template = self.__rendered_template(self.device, self.config_context, self.device_template) + + def __rendered_template(self, device, config_context, device_template): + if len(config_context) >0: + template_directory = os.path.dirname(device_template) + template_file = os.path.basename(device_template) + # rendered_template_file = f'{device}.txt' + environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_directory)) + environment.trim_blocks = True + environment.lstrip_blocks = True + # add config_context dict to a global in the rendering environment, access variables like golden config + environment.globals['config_context'] = config_context + # pretty_config_context is for illustration only, non functional variable as keys cannot be accessed + environment.globals['pretty_config_context'] = yaml.dump(config_context, default_flow_style=0) + template = environment.get_template(template_file) + # render without specifying variables or a dict as input, instead use environment.globals['config_context'] + rendered_template_content = template.render() + # rendered_template_content = template.render(config_context) + return rendered_template_content + else: + print(f'==== device template render issue: {device}\n\n{os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context {config_context} (clashing top level keys in config_contexts)\n') + return + + def print_rendered_template(self): + if self.rendered_template is not None: + print(f'==== {self.device} file content:\n\n{self.rendered_template}\n') + + def write_rendered_template(self): + if self.rendered_template is not None: + # this needs more logic to pass another parameter for dest dir + rendered_template_file = f'{self.device}.txt' + with open(rendered_template_file, mode="w", encoding="utf-8") as message: + message.write(self.rendered_template) + print(f'==== {self.device} file written: {rendered_template_file}') + +def main(): + config_contexts_dir = './config_contexts' + templates_dir = './templates' + inventory_file = "./inventory.yml" + inventory = LoadInventory(config_contexts_dir, templates_dir, inventory_file) + # print(inventory.get_devices()) + # print(yaml.dump(inventory.get_contexts())) + + config_contexts_list = [] + for i in inventory.get_devices(): + context_obj = RenderDeviceConfig(inventory, i) + if len(context_obj.get_config_context()) >0: + # print(f'device:\n{context_obj.get_device_name()}\n') + # print(f'config_context:\n{yaml.dump(context_obj.get_config_context())}') + config_contexts_list.append(context_obj) + else: + del context_obj + + for i in config_contexts_list: + i.print_rendered_template() + i.write_rendered_template() + +if __name__ == "__main__": + main() + +# turn this into a tool? +# add comment block to each renderd config such as rendered date (maybe list templates in use?) +# needs a config file to point to contexts/folders, use toml top allow for comments +# needs a rendered artefact directory parameter somewhere, maybe this should be a write_rendered_template parameter to allow it to be placed in another git repo +# needs proper logging +# needs a mode to create example files/dirs diff --git a/templates/eos.j2 b/templates/eos.j2 new file mode 100644 index 0000000..193d59c --- /dev/null +++ b/templates/eos.j2 @@ -0,0 +1,70 @@ +**** start of template **** + +0) display the config_context: + + top level context item: + + {{ config_context }} + + selecting the best unicorn with config_context['best_unicorn'][0]: + + {{ config_context['best_unicorn'][0] }} + + accessing a key pair injected into the jinja environment global variables: + (in this case the config_context printed out in human readable yaml format) + + {% filter indent(width=3) %} + {{ pretty_config_context }} + {% endfilter %} + + loop through the config_context variable in jinja: + + {% for key, value in config_context.items() %} + {{ key }}:{{ value }} + {% endfor %} + +1) source template ./device/{{ config_context['device'] }}.yml + +2) source template ./device/region/{{ config_context['region'] }}.yml + +3) source template ./device/site/{{ config_context['site'] }}.yml + +4) source template ./device/role/.yml + these contexts happens to be a list in the inventory + files under this directory happen to have the same weight(they dont have to) + thus these files do not have the same keys as they would clash + + {% if config_context['gabbys_doll_house'] %} + gabbys_doll_house = {{ config_context['gabbys_doll_house'] }} + {% endif %} + + {% if config_context['octonaughts'] %} + octonaughts = {{ config_context['octonaughts'] }} + {% endif %} + + notice best_unicorn is in the all.yml context and router.yml context + router.yml has a higher precedence(lower weight value) + + {% if config_context['best_unicorn'] %} + {% for entry in config_context['best_unicorn'] %} + - {{ entry }} + {% endfor %} + {% endif %} + + {% if config_context['harry_potter'] %} + harry_potter = {{ config_context['harry_potter'] }} + {% endif %} + + notice cbeebies_schedule is in the all.yml context and vpn.yml context + all.yml context is weighted last/-1, any other context will overwrite it + vpn.yml has a higher precedence(lower weight value) + + {% if config_context['cbeebies_schedule'] %} + cbeebies_schedule = {{ config_context['cbeebies_schedule'] }} + {% endif %} + +4) source template ./device/role/all.yml + nothing is rendered using this context, all variables are overwritten by contexts with higher precedence + this would be a good place to put service accounts in ACLs or legal disclaimers + +**** end of template **** \ No newline at end of file diff --git a/templates/ios.j2 b/templates/ios.j2 new file mode 100644 index 0000000..97f4e0d --- /dev/null +++ b/templates/ios.j2 @@ -0,0 +1 @@ +{{ cbeebies_schedule }} \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..722b127 --- /dev/null +++ b/test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +import os +import sys +import datetime +import yaml + +def contexts(device, config_contexts): + # print(f'device: {device}') + all_config_contexts = {} + + def remove_meta(data): + try: + del data['_metadata'] + except KeyError: + pass + return data + + # get device context + try: + with open(f'./config_contexts/device/{device}.yml', 'r') as file: + data = yaml.safe_load(file) + all_config_contexts.update({'device': {'weight': 0, 'content': remove_meta(data)}}) + except: + pass + + # get config_contexts + for k, v in config_contexts.items(): + islist = False + if isinstance(v, list): + islist = True + vlist = v + else: + vlist = [v] + for i in vlist: + try: + if islist: + context_name = f'{k}_{i}' + else: + context_name = k + with open(f'./config_contexts/{k}/{i}.yml', 'r') as file: + data = yaml.safe_load(file) + if data['_metadata']['weight'] and data['_metadata']['is_active']: + all_config_contexts.update({context_name: {'weight': data['_metadata']['weight'], 'content': remove_meta(data)}}) + except: + pass + + # get the all context + all_weight = sorted([all_config_contexts[k]['weight'] for k, v in all_config_contexts.items()], reverse=True) + dupe_weight = list(set([x for x in all_weight if all_weight.count(x) > 1])) + max_weight = all_weight[0]+1 + try: + with open('./config_contexts/all.yml', 'r') as file: + data = yaml.safe_load(file) + if data['_metadata']['is_active']: + all_config_contexts.update({'all': {'weight': max_weight, 'content': remove_meta(data)}}) + except: + pass + # print(yaml.dump(all_config_contexts)) + + # check for equally weighted clashing keys + equal_weight = {} + for i in dupe_weight: + equal_weight.update({i: []}) + for k in all_config_contexts.keys(): + if all_config_contexts[k]['weight'] == i: + equal_weight[i].append(k) + # print(f'equal weight contexts: {equal_weight}') + + # list all keys in equally weighted contexts + keys_list = [] + for k in equal_weight.keys(): + for i in equal_weight[k]: + for k in all_config_contexts[i]['content'].keys(): + keys_list.append(k) + keys_list = list(set(keys_list)) + # print(f'all keys in equal weight contexts: {keys_list}') + + # check each equally weighted context for clashing keys + equal_weight_key_clash_errors = [] + for w in equal_weight: + for k in keys_list: + key_occurance = [] + for i in equal_weight[w]: + if k in all_config_contexts[i]['content']: + key_occurance.append(i) + if len(key_occurance) >1: + equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}') + if len(equal_weight_key_clash_errors) >0: + error_log = f'device config_context render issue: {device}\n' + for i in equal_weight_key_clash_errors: + error_log = error_log + f'\n{i}\n' + print(error_log) + return + + # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting keys where they exist (the last context being the lowest weight and highest precedence) + config_context = {} + all_config_contexts = {k: v for k, v in sorted(all_config_contexts.items(), key=lambda x: x[1]['weight'], reverse=True)} + for k, v in all_config_contexts.items(): + config_context = {**config_context, **v['content']} + config_context = dict(sorted(config_context.items())) + print(yaml.dump(config_context)) + return config_context + +def main(): + with open('./inventory.yml', 'r') as file: + data = yaml.safe_load(file) + for i in data.keys(): + contexts(i, data[i]['config_contexts']) + +if __name__ == "__main__": + main() + + # do a validation class + # do a device class - this should be added to a list that the render class does something with + + + # what does this need to do? + # render a config_context + # render a jinja2 template + # print out a rendered context for someone to develop a jinja2 template + + # you can easily wrap this up as a class, it just needs to be called with some paths and have methods to return the config_contexts + # this really needs to be changed to validate all configs first, then start ingesting devices \ No newline at end of file