init

.gitignore

@@ -0,0 +1 @@
+venv_nautofake/

config_contexts/all.yml

@@ -0,0 +1,11 @@
+---
+_metadata:
+  name: "all"
+  description: "Global Definitions"
+  is_active: true
+cbeebies_schedule: "16/11/2023"
+best_unicorn:
+  - sparkle
+  - "star light"
+  - "glitter love"
+  - twinkle

config_contexts/device/device_a.yml

@@ -0,0 +1 @@
+device: device_a

config_contexts/device/device_b.yml

@@ -0,0 +1 @@
+device: device_b

config_contexts/region/asia.yml

@@ -0,0 +1,6 @@
+_metadata:
+  name: "all"
+  weight: 200
+  description: "Region Asia Definitions"
+  is_active: true
+region: asia

config_contexts/region/europe.yml

@@ -0,0 +1,6 @@
+_metadata:
+  name: "all"
+  weight: 200
+  description: "Region Europe Definitions"
+  is_active: true
+region: europe

config_contexts/region/north_america.yml

@@ -0,0 +1,6 @@
+_metadata:
+  name: "all"
+  weight: 200
+  description: "Region North America Definitions"
+  is_active: true
+region: north_america

config_contexts/region/south_america.yml

@@ -0,0 +1,6 @@
+_metadata:
+  name: "all"
+  weight: 200
+  description: "Region South America Definitions"
+  is_active: true
+region: south_america

config_contexts/role/router.yml

@@ -0,0 +1,13 @@
+_metadata:
+  name: "all"
+  weight: 400
+  description: "Router Definitions"
+  is_active: true
+octonaughts: 1
+gabbys_doll_house: 1
+best_unicorn:
+  - "sugar socks"
+  - "star light"
+  - sparkle
+  - twinkle
+  - "glitter love"

config_contexts/role/switch.yml

@@ -0,0 +1,7 @@
+_metadata:
+  name: "all"
+  weight: 400
+  description: "Switch Definitions"
+  is_active: true
+harry_potter: 1
+gabbys_doll_house: 1

config_contexts/role/vpn.yml

@@ -0,0 +1,7 @@
+_metadata:
+  name: "all"
+  weight: 400
+  description: "VPN Definitions"
+  is_active: true
+harry_potter: 1
+cbeebies_schedule: "17/11/2023"

config_contexts/site/dub.yml

@@ -0,0 +1,7 @@
+_metadata:
+  name: "all"
+  weight: 300
+  description: "Site Dublin Definitions"
+  is_active: true
+site: dublin
+cbeebies_schedule: "18/11/2023"

config_contexts/site/lon.yml

@@ -0,0 +1,6 @@
+_metadata:
+  name: "all"
+  weight: 300
+  description: "Site London Definitions"
+  is_active: true
+site: london

inventory.yml

@@ -0,0 +1,59 @@
+# the inventory file dictates the content of the config_context dictionary item used to render jinja2 templates
+# this could be generated by a csv, populated in a database or simply exist as a flat file
+
+# content of inventory.yml
+#
+# router1_customer1:     # include variables from config_contexts/device/router1_customer1.yml
+#   platform: eos        # include template from templates/eos.j2
+#   config_contexts:     # create folders by the name of the key and files by the name of the key value(s), 
+#       site: dub        # include variables from config_contexts/site/dub.yml
+#       region: europe
+#       role:            # include variables from config_contexts/role/router.yml and config_contexts/role/vpn.yml, list may contain any number of items
+#         - router
+#         - vpn
+
+# content of config_contexts/device/router1_customer1.yml
+# there are no special/functional items
+# the device context has weight 0, top precedence
+# keys in this file will overwrite any duplicate keys imported from any other context. 
+#
+# key_name: "some_value"
+
+# content of files under the context_contexts folder, example config_contexts/site/dub.yml
+# the _metadata object, keys 'weight'(int) and is_active(bool) are functional, they control the order of precedence in which the keys are merged to the config_context
+#
+# _metadata:
+#   name: "dublin"
+#   weight: 300
+#   description: "Site Dublin Definitions"
+#   is_active: true
+# site: dublin
+# cbeebies_schedule: "18/11/2023"
+
+device_a:
+  platform: ios
+  config_contexts:
+      region: europe
+      site: lon
+      role:
+        - router
+        - switch
+        - vpn
+        - firewall
+device_b:
+  platform: eos
+  config_contexts:
+      site: dub
+      region: europe
+      role:
+        - router
+        - vpn
+# device_c:
+#   config_contexts:
+#       region: asia
+#       platform: ios
+# device_d:
+#   config_contexts:
+#       region: south_america
+
+

python_venv.md

@@ -0,0 +1,13 @@
+# Python venv
+
+- Host with Python version 3.6+
+
+```sh
+python3 -m venv $HOME/nautofake/venv_nautofake
+source $HOME/nautofake/venv_nautofake/bin/activate
+python --version
+which python
+pip install --upgrade pip
+pip install pyyaml jinja2
+deactivate
+'''

render_engine.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python
+
+import sys
+from pathlib import Path
+import os
+import yaml
+import jinja2
+
+class FileChecks:
+    'file check rules'
+    def __init__(self, config_contexts_dir, templates_dir, inventory_file):
+      self.config_contexts_dir = config_contexts_dir
+      self.templates_dir = templates_dir
+      self.inventory_file = inventory_file
+      self.inventory_dict = self.__read_inventory_file(self.inventory_file)
+      self.__check_context_files(self.config_contexts_dir)
+      self.__check_platform_templates(self.inventory_dict, self.templates_dir)
+
+    def __read_inventory_file(self, inventory_file):
+        try:
+          with open(inventory_file, "r") as file:
+            return yaml.safe_load(file)
+        except Exception as ex:
+          if type(ex).__name__ == 'ScannerError':
+            print(f'Error {inventory_file}: is not valid YAML')
+            sys.exit(1)
+          elif type(ex).__name__ ==  'FileNotFoundError':
+            print(f'Error {inventory_file}: {ex.args[1]}')
+            sys.exit(1)
+          else:
+            print(f'Error: {inventory_file}')
+            sys.exit(1)
+
+    def __check_context_files(self, config_contexts_dir):
+        context_files = {}
+        # check for contexts directory - need an all file, dont need a device file
+        for path in Path(config_contexts_dir).rglob('*.yml'):
+          context_files.update({str(path): { 'file': path.name,'path': str(path.parent)}})
+        file_names = [context_files[i]['file'] for i in context_files.keys()]
+        duplicate_files = list(set([i for i in file_names if file_names.count(i) > 1]))
+        if len(duplicate_files) >0:
+          print('==== to maintain readability there no logic to handle duplicate file names under the config_contexts directory')
+          for i in duplicate_files:
+             for j in context_files.keys():
+                if context_files[j]['file'] == i:
+                   print(f"duplicate file name {context_files[j]['file']} @ {j}")
+          sys.exit(1)
+
+    def __check_platform_templates(self, inventory_dict, templates_dir):
+        platforms = set([inventory_dict[i]['platform'] for i in inventory_dict.keys()])
+        for i in platforms:
+          template = Path(f'{templates_dir}/{i}.j2')
+          if not template.is_file():
+            target_device_list = [k for k, v in inventory_dict.items() if v['platform'] == i]
+            target_devices = '\n'.join(target_device_list)
+            print(f'==== missing template {template}, nothing will be rendered for devices:\n{target_devices}')
+
+class LoadInventory(FileChecks):
+    'load inventory'
+    def __init__(self, config_contexts_dir, templates_dir, inventory_file):
+        FileChecks.__init__(self, config_contexts_dir, templates_dir, inventory_file)
+        self.all_config_contexts, self.devices = self.__load_contexts(self.inventory_dict)
+        #print(yaml.dump(self.all_config_contexts))
+
+    def __remove_meta(self, data):
+      try:
+        del data['_metadata']
+      except KeyError:
+        pass
+      return data
+
+    def __load_contexts(self, inventory_dict):
+      # get path of all device and context files
+      devices_path = set()
+      contexts_path = set()
+      devices = []
+      for i in inventory_dict.keys():
+        devices_path.add(f'{self.config_contexts_dir}/device/{i}.yml')
+        devices.append(i)
+        for k, v in inventory_dict[i]['config_contexts'].items():      
+          if isinstance(v, list):
+            for i in v:
+              contexts_path.add(f'{self.config_contexts_dir}/{k}/{i}.yml')
+          else:
+            contexts_path.add(f'{self.config_contexts_dir}/{k}/{v}.yml')
+      # print(devices_path)
+      # print(contexts_path)
+
+      # load config contexts
+      all_config_contexts = {}
+      for i in contexts_path:
+        try:
+          with open(i, 'r') as file:
+            data = yaml.safe_load(file)
+          if data['_metadata']['weight'] and data['_metadata']['is_active']:
+            # build the context_name from the file parent directory
+            # context naming could be sourced from the metadata dict but the potential for duplicate filenames under config_contexts subdirectories is to be discouraged
+            context = os.path.splitext(os.path.basename(i))[0]
+            all_config_contexts.update({context: {'weight': data['_metadata']['weight'], 'content': self.__remove_meta(data)}})
+        except:
+          pass
+      # print(all_config_contexts)
+
+      # load device contexts
+      for i in devices_path:
+        try:
+          with open(i, 'r') as file:
+            data = yaml.safe_load(file)
+          context = os.path.splitext(os.path.basename(i))[0]
+          all_config_contexts.update({context: {'weight': 0, 'content': self.__remove_meta(data)}})
+        except:
+          pass
+
+      # load all contexts
+      all_weight = sorted([all_config_contexts[k]['weight'] for k, v in all_config_contexts.items()], reverse=True)
+      max_weight = all_weight[0]+1
+      try:
+        with open(f'{self.config_contexts_dir}/all.yml', 'r') as file:
+          data = yaml.safe_load(file)
+          if data['_metadata']['is_active']:
+            all_config_contexts.update({'all': {'weight': max_weight, 'content': self.__remove_meta(data)}})
+      except:
+        pass
+
+      return all_config_contexts, devices
+
+    def get_contexts(self):
+      return self.all_config_contexts
+
+    def get_devices(self):
+      return self.devices
+
+    def get_device_platform(self, device):
+      return self.inventory_dict[device]['platform']
+
+    def get_device_template(self, device):
+      platform = self.get_device_platform(device)
+      return f'{self.templates_dir}/{platform}.j2'
+
+    def get_device_contexts(self, device):
+      device_contexts = {}
+      all_device_contexts = ['all', device]
+      for i in all_device_contexts:
+        if i in self.all_config_contexts:
+          device_contexts.update({i: self.all_config_contexts[i]})
+      for k, v in self.inventory_dict[device]['config_contexts'].items():  
+        if isinstance(v, list):
+          for i in v:
+            if i in self.all_config_contexts:
+              device_contexts.update({i: self.all_config_contexts[i]})
+        elif v in self.all_config_contexts:
+          device_contexts.update({v: self.all_config_contexts[v]})
+      return device_contexts
+
+class BuildDeviceContext:
+    'build config_context for a device'
+    def __init__(self, LoadInventory_instance, device):
+      self.device = device
+      self.contexts = LoadInventory_instance.get_device_contexts(self.device)
+      self.platform = LoadInventory_instance.get_device_platform(self.device)
+      # print(self.contexts)
+      # print(self.platform)
+      self.config_context = self.__config_context(self.device, self.contexts)
+
+    def __config_context(self, device, contexts):
+        # check for equally weighted clashing keys
+        all_weight = sorted([contexts[k]['weight'] for k, v in contexts.items()], reverse=True)
+        dupe_weight = list(set([x for x in all_weight if all_weight.count(x) > 1]))
+        equal_weight = {}
+        for i in dupe_weight:
+          equal_weight.update({i: []})
+          for k in contexts.keys():
+            if contexts[k]['weight'] == i:
+              equal_weight[i].append(k)
+        # print(f'equal weight contexts: {equal_weight}')
+  
+        # list all keys in equally weighted contexts
+        keys_list = []
+        for k in equal_weight.keys():
+          for i in equal_weight[k]:
+            for k in contexts[i]['content'].keys():
+              keys_list.append(k)
+        keys_list = list(set(keys_list))
+        # print(f'all keys in equal weight contexts: {keys_list}')
+
+        # check each equally weighted context for clashing keys
+        equal_weight_key_clash_errors = []
+        for w in equal_weight:
+          for k in keys_list:
+            key_occurance = []
+            for i in equal_weight[w]:
+              if k in contexts[i]['content']:
+                key_occurance.append(i)
+            if len(key_occurance) >1:
+              equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}')
+        if len(equal_weight_key_clash_errors) >0:
+          error_log = f'==== device config_context render issue: {device}\n'
+          for i in equal_weight_key_clash_errors:
+            error_log = error_log + f'\n{i}\n'
+          print(error_log)
+          return {}
+
+        # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting top level keys where they exist (the last context being the lowest weight and highest precedence)
+        config_context = {}
+        contexts = {k: v for k, v in sorted(contexts.items(), key=lambda x: x[1]['weight'], reverse=True)}
+        for k, v in contexts.items():
+          config_context = {**config_context, **v['content']}
+        config_context = dict(sorted(config_context.items()))
+        # print(yaml.dump(config_context))
+        return config_context
+
+    def get_config_context(self):
+      return self.config_context
+
+    def get_device_name(self):
+      return self.device
+
+class RenderDeviceConfig(BuildDeviceContext):
+    'build config_context for a device'
+    def __init__(self, LoadInventory_instance, device):
+        BuildDeviceContext.__init__(self, LoadInventory_instance, device)
+        self.device_template = LoadInventory_instance.get_device_template(self.device)
+        self.rendered_template = self.__rendered_template(self.device, self.config_context, self.device_template)
+
+    def __rendered_template(self, device, config_context, device_template):
+        if len(config_context) >0:
+          template_directory = os.path.dirname(device_template)
+          template_file = os.path.basename(device_template)
+          # rendered_template_file = f'{device}.txt'
+          environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_directory))
+          environment.trim_blocks = True
+          environment.lstrip_blocks = True
+          # add config_context dict to a global in the rendering environment, access variables like golden config
+          environment.globals['config_context'] = config_context
+          # pretty_config_context is for illustration only, non functional variable as keys cannot be accessed
+          environment.globals['pretty_config_context'] = yaml.dump(config_context, default_flow_style=0)
+          template = environment.get_template(template_file)
+          # render without specifying variables or a dict as input, instead use environment.globals['config_context']
+          rendered_template_content = template.render()
+          # rendered_template_content = template.render(config_context)
+          return rendered_template_content
+        else:
+          print(f'==== device template render issue: {device}\n\n{os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context {config_context} (clashing top level keys in config_contexts)\n')
+          return
+
+    def print_rendered_template(self):
+        if self.rendered_template is not None:
+          print(f'==== {self.device} file content:\n\n{self.rendered_template}\n')
+
+    def write_rendered_template(self):
+        if self.rendered_template is not None:
+          # this needs more logic to pass another parameter for dest dir
+          rendered_template_file = f'{self.device}.txt'
+          with open(rendered_template_file, mode="w", encoding="utf-8") as message:
+              message.write(self.rendered_template)
+              print(f'==== {self.device} file written: {rendered_template_file}')
+
+def main():
+  config_contexts_dir = './config_contexts'
+  templates_dir = './templates'
+  inventory_file = "./inventory.yml"
+  inventory = LoadInventory(config_contexts_dir, templates_dir, inventory_file)
+  # print(inventory.get_devices())
+  # print(yaml.dump(inventory.get_contexts()))
+
+  config_contexts_list = []
+  for i in inventory.get_devices():
+    context_obj = RenderDeviceConfig(inventory, i)
+    if len(context_obj.get_config_context()) >0:
+      # print(f'device:\n{context_obj.get_device_name()}\n')
+      # print(f'config_context:\n{yaml.dump(context_obj.get_config_context())}')
+      config_contexts_list.append(context_obj)
+    else:
+      del context_obj
+
+  for i in config_contexts_list:
+    i.print_rendered_template()
+    i.write_rendered_template()
+
+if __name__ == "__main__":
+    main()
+
+# turn this into a tool?
+# add comment block to each renderd config such as rendered date (maybe list templates in use?)
+# needs a config file to point to contexts/folders, use toml top allow for comments
+# needs a rendered artefact directory parameter somewhere, maybe this should be a write_rendered_template parameter to allow it to be placed in another git repo
+# needs proper logging
+# needs a mode to create example files/dirs

templates/eos.j2

@@ -0,0 +1,70 @@
+**** start of template ****
+
+0) display the config_context:
+
+   top level context item:
+
+   {{ config_context }}
+
+   selecting the best unicorn with config_context['best_unicorn'][0]:
+
+   {{ config_context['best_unicorn'][0] }}
+
+   accessing a key pair injected into the jinja environment global variables:
+   (in this case the config_context printed out in human readable yaml format)
+
+   {% filter indent(width=3) %}
+   {{ pretty_config_context }}
+   {% endfilter %}
+
+   loop through the config_context variable in jinja:
+
+   {% for key, value in config_context.items() %}
+   {{ key }}:{{ value }}
+   {% endfor %}
+
+1) source template ./device/{{ config_context['device'] }}.yml
+
+2) source template ./device/region/{{ config_context['region'] }}.yml
+
+3) source template ./device/site/{{ config_context['site'] }}.yml
+
+4) source template ./device/role/<files>.yml
+   these contexts happens to be a list in the inventory
+   files under this directory happen to have the same weight(they dont have to)
+   thus these files do not have the same keys as they would clash
+
+   {% if config_context['gabbys_doll_house'] %}
+   gabbys_doll_house = {{ config_context['gabbys_doll_house'] }}
+   {% endif %}
+
+   {% if config_context['octonaughts'] %}
+   octonaughts = {{ config_context['octonaughts'] }}
+   {% endif %}
+
+   notice best_unicorn is in the all.yml context and router.yml context
+   router.yml has a higher precedence(lower weight value)
+
+   {% if config_context['best_unicorn'] %}
+   {% for entry in config_context['best_unicorn'] %}
+   - {{ entry }}
+   {% endfor %}
+   {% endif %}
+
+   {% if config_context['harry_potter'] %}
+   harry_potter = {{ config_context['harry_potter'] }}
+   {% endif %}
+   
+   notice cbeebies_schedule is in the all.yml context and vpn.yml context
+   all.yml context is weighted last/-1, any other context will overwrite it
+   vpn.yml has a higher precedence(lower weight value)
+
+   {% if config_context['cbeebies_schedule'] %}
+   cbeebies_schedule = {{ config_context['cbeebies_schedule'] }}
+   {% endif %}
+
+4) source template ./device/role/all.yml
+   nothing is rendered using this context, all variables are overwritten by contexts with higher precedence
+   this would be a good place to put service accounts in ACLs or legal disclaimers
+
+**** end of template ****

templates/ios.j2

@@ -0,0 +1 @@
+{{ cbeebies_schedule }}

test.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import datetime
+import yaml
+
+def contexts(device, config_contexts):
+  # print(f'device: {device}')
+  all_config_contexts = {}
+
+  def remove_meta(data):
+    try:
+      del data['_metadata']
+    except KeyError:
+      pass
+    return data
+
+  # get device context
+  try:
+    with open(f'./config_contexts/device/{device}.yml', 'r') as file:
+      data = yaml.safe_load(file)
+      all_config_contexts.update({'device': {'weight': 0, 'content': remove_meta(data)}})
+  except:
+    pass
+
+  # get config_contexts
+  for k, v in config_contexts.items():
+    islist = False
+    if isinstance(v, list):
+      islist = True
+      vlist = v
+    else:
+      vlist = [v]
+    for i in vlist:
+      try:
+        if islist:
+          context_name = f'{k}_{i}'
+        else:
+          context_name = k
+        with open(f'./config_contexts/{k}/{i}.yml', 'r') as file:
+          data = yaml.safe_load(file)
+        if data['_metadata']['weight'] and data['_metadata']['is_active']:
+            all_config_contexts.update({context_name: {'weight': data['_metadata']['weight'], 'content': remove_meta(data)}})
+      except:
+        pass
+
+  # get the all context
+  all_weight = sorted([all_config_contexts[k]['weight'] for k, v in all_config_contexts.items()], reverse=True)
+  dupe_weight = list(set([x for x in all_weight if all_weight.count(x) > 1]))
+  max_weight = all_weight[0]+1
+  try:
+    with open('./config_contexts/all.yml', 'r') as file:
+      data = yaml.safe_load(file)
+      if data['_metadata']['is_active']:
+        all_config_contexts.update({'all': {'weight': max_weight, 'content': remove_meta(data)}})
+  except:
+    pass  
+  # print(yaml.dump(all_config_contexts))
+
+  # check for equally weighted clashing keys
+  equal_weight = {}
+  for i in dupe_weight:
+    equal_weight.update({i: []})
+    for k in all_config_contexts.keys():
+      if all_config_contexts[k]['weight'] == i:
+        equal_weight[i].append(k)
+  # print(f'equal weight contexts: {equal_weight}')
+  
+  # list all keys in equally weighted contexts
+  keys_list = []
+  for k in equal_weight.keys():
+    for i in equal_weight[k]:
+      for k in all_config_contexts[i]['content'].keys():
+        keys_list.append(k)
+  keys_list = list(set(keys_list))
+  # print(f'all keys in equal weight contexts: {keys_list}')
+
+  # check each equally weighted context for clashing keys
+  equal_weight_key_clash_errors = []
+  for w in equal_weight:
+    for k in keys_list:
+      key_occurance = []
+      for i in equal_weight[w]:
+        if k in all_config_contexts[i]['content']:
+          key_occurance.append(i)
+      if len(key_occurance) >1:
+        equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}')
+  if len(equal_weight_key_clash_errors) >0:
+    error_log = f'device config_context render issue: {device}\n'
+    for i in equal_weight_key_clash_errors:
+      error_log = error_log + f'\n{i}\n'
+    print(error_log)
+    return
+
+  # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting keys where they exist (the last context being the lowest weight and highest precedence)
+  config_context = {}
+  all_config_contexts = {k: v for k, v in sorted(all_config_contexts.items(), key=lambda x: x[1]['weight'], reverse=True)}
+  for k, v in all_config_contexts.items():
+    config_context = {**config_context, **v['content']}
+  config_context = dict(sorted(config_context.items()))
+  print(yaml.dump(config_context))
+  return config_context
+
+def main():
+  with open('./inventory.yml', 'r') as file:
+    data = yaml.safe_load(file)
+  for i in data.keys():
+    contexts(i, data[i]['config_contexts'])
+
+if __name__ == "__main__":
+    main()
+
+  # do a validation class
+  # do a device class - this should be added to a list that the render class does something with
+
+
+  # what does this need to do?
+  # render a config_context
+  # render a jinja2 template
+  # print out a rendered context for someone to develop a jinja2 template
+
+  # you can easily wrap this up as a class, it just needs to be called with some paths and have methods to return the config_contexts
+  # this really needs to be changed to validate all configs first, then start ingesting devices