logging + config file

README.md

@@ -14,7 +14,7 @@ source $HOME/gc_integration_templater/venv/bin/activate
 python --version
 which python
 pip install --proxy http://asblcinfpxy01:3128 --upgrade pip 
-pip install --proxy http://asblcinfpxy01:3128 pyyaml jinja2
+pip install --proxy http://asblcinfpxy01:3128 pyyaml jinja2 toml
 deactivate
 ```
 

config.toml

@@ -0,0 +1,4 @@
+config_contexts_dir = './config_contexts'
+platform_templates_dir = './platform_templates'
+rendered_templates_dir = './rendered_templates'
+inventory_file = './inventory.yml'

inventory.yml

@@ -4,7 +4,7 @@
 # content of inventory.yml
 #
 # router1_customer1:     # include variables from config_contexts/device/router1_customer1.yml
-#   platform: eos        # include template from templates/eos.j2
+#   platform: eos        # include template from platform_templates/eos.j2
 #   config_contexts:     # create folders by the name of the key and files by the name of the key value(s), 
 #       site: dub        # include variables from config_contexts/site/dub.yml
 #       region: europe

render_engine.log

@@ -0,0 +1,214 @@
+2023-12-12 11:35:22, ERROR    | render_engine.py:304   | MainThread: device config_context render issue: device_a
+clashing key found in equally weighted contexts: 
+key: harry_potter 
+contexts: ['switch', 'vpn']
+
+clashing key found in equally weighted contexts: 
+key: gabbys_doll_house 
+contexts: ['router', 'switch']
+
+2023-12-12 11:35:22, ERROR    | render_engine.py:350   | MainThread: device template render issue: device_a
+template: ./platform_templates/ios.j2
+empty config_context: {}
+2023-12-12 11:35:22, INFO     | render_engine.py:364   | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt
+2023-12-12 11:35:22, INFO     | render_engine.py:356   | MainThread: device_b rendered file content:
+
+**** start of template ****
+
+0) display the config_context:
+
+   top level context item:
+
+   {'best_unicorn': ['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'], 'cbeebies_schedule': '18/11/2023', 'device': 'device_b', 'gabbys_doll_house': 1, 'harry_potter': 1, 'octonaughts': 1, 'region': 'europe', 'site': 'dublin'}
+
+   selecting the best unicorn with config_context['best_unicorn'][0]:
+
+   sugar socks
+
+   accessing a key pair injected into the jinja environment global variables:
+   (in this case the config_context printed out in human readable yaml format)
+
+   best_unicorn:
+   - sugar socks
+   - star light
+   - sparkle
+   - twinkle
+   - glitter love
+   cbeebies_schedule: 18/11/2023
+   device: device_b
+   gabbys_doll_house: 1
+   harry_potter: 1
+   octonaughts: 1
+   region: europe
+   site: dublin
+
+
+   loop through the config_context variable in jinja:
+
+   best_unicorn:['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love']
+   cbeebies_schedule:18/11/2023
+   device:device_b
+   gabbys_doll_house:1
+   harry_potter:1
+   octonaughts:1
+   region:europe
+   site:dublin
+
+1) source template ./device/device_b.yml
+
+2) source template ./device/region/europe.yml
+
+3) source template ./device/site/dublin.yml
+
+4) source template ./device/role/<files>.yml
+   these contexts happens to be a list in the inventory
+   files under this directory happen to have the same weight(they dont have to)
+   thus these files do not have the same keys as they would clash
+
+   gabbys_doll_house = 1
+
+   octonaughts = 1
+
+   notice best_unicorn is in the all.yml context and router.yml context
+   router.yml has a higher precedence(lower weight value)
+
+   - sugar socks
+   - star light
+   - sparkle
+   - twinkle
+   - glitter love
+
+   harry_potter = 1
+   
+   notice cbeebies_schedule is in the all.yml context and vpn.yml context
+   all.yml context is weighted last/-1, any other context will overwrite it
+   vpn.yml has a higher precedence(lower weight value)
+
+   cbeebies_schedule = 18/11/2023
+
+4) source template ./device/role/all.yml
+   nothing is rendered using this context, all variables are overwritten by contexts with higher precedence
+   this would be a good place to put service accounts in ACLs or legal disclaimers
+
+**** end of template ****
+
+2023-12-12 11:36:41, ERROR    | render_engine.py:304   | MainThread: device config_context render issue: device_a
+clashing key found in equally weighted contexts: 
+key: gabbys_doll_house 
+contexts: ['router', 'switch']
+
+clashing key found in equally weighted contexts: 
+key: harry_potter 
+contexts: ['switch', 'vpn']
+
+2023-12-12 11:36:41, ERROR    | render_engine.py:350   | MainThread: device template render issue: device_a
+template: ./platform_templates/ios.j2
+empty config_context: {}
+2023-12-12 11:36:41, INFO     | render_engine.py:364   | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt
+2023-12-12 11:36:41, INFO     | render_engine.py:356   | MainThread: device_b rendered file content:
+
+**** start of template ****
+
+0) display the config_context:
+
+   top level context item:
+
+   {'best_unicorn': ['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'], 'cbeebies_schedule': '18/11/2023', 'device': 'device_b', 'gabbys_doll_house': 1, 'harry_potter': 1, 'octonaughts': 1, 'region': 'europe', 'site': 'dublin'}
+
+   selecting the best unicorn with config_context['best_unicorn'][0]:
+
+   sugar socks
+
+   accessing a key pair injected into the jinja environment global variables:
+   (in this case the config_context printed out in human readable yaml format)
+
+   best_unicorn:
+   - sugar socks
+   - star light
+   - sparkle
+   - twinkle
+   - glitter love
+   cbeebies_schedule: 18/11/2023
+   device: device_b
+   gabbys_doll_house: 1
+   harry_potter: 1
+   octonaughts: 1
+   region: europe
+   site: dublin
+
+
+   loop through the config_context variable in jinja:
+
+   best_unicorn:['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love']
+   cbeebies_schedule:18/11/2023
+   device:device_b
+   gabbys_doll_house:1
+   harry_potter:1
+   octonaughts:1
+   region:europe
+   site:dublin
+
+1) source template ./device/device_b.yml
+
+2) source template ./device/region/europe.yml
+
+3) source template ./device/site/dublin.yml
+
+4) source template ./device/role/<files>.yml
+   these contexts happens to be a list in the inventory
+   files under this directory happen to have the same weight(they dont have to)
+   thus these files do not have the same keys as they would clash
+
+   gabbys_doll_house = 1
+
+   octonaughts = 1
+
+   notice best_unicorn is in the all.yml context and router.yml context
+   router.yml has a higher precedence(lower weight value)
+
+   - sugar socks
+   - star light
+   - sparkle
+   - twinkle
+   - glitter love
+
+   harry_potter = 1
+   
+   notice cbeebies_schedule is in the all.yml context and vpn.yml context
+   all.yml context is weighted last/-1, any other context will overwrite it
+   vpn.yml has a higher precedence(lower weight value)
+
+   cbeebies_schedule = 18/11/2023
+
+4) source template ./device/role/all.yml
+   nothing is rendered using this context, all variables are overwritten by contexts with higher precedence
+   this would be a good place to put service accounts in ACLs or legal disclaimers
+
+**** end of template ****
+
+2023-12-12 11:36:55, ERROR    | render_engine.py:303   | MainThread: device config_context render issue: device_a
+clashing key found in equally weighted contexts: 
+key: gabbys_doll_house 
+contexts: ['router', 'switch']
+
+clashing key found in equally weighted contexts: 
+key: harry_potter 
+contexts: ['switch', 'vpn']
+
+2023-12-12 11:36:55, ERROR    | render_engine.py:349   | MainThread: device template render issue: device_a
+template: ./platform_templates/ios.j2
+empty config_context: {}
+2023-12-12 11:36:55, INFO     | render_engine.py:363   | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt
+2023-12-12 11:46:39, ERROR    | render_engine.py:291   | MainThread: device config_context render issue: device_a
+clashing key found in equally weighted contexts: 
+key: harry_potter 
+contexts: ['switch', 'vpn']
+
+clashing key found in equally weighted contexts: 
+key: gabbys_doll_house 
+contexts: ['router', 'switch']
+
+2023-12-12 11:46:39, ERROR    | render_engine.py:335   | MainThread: device template render issue: device_a
+template: ./platform_templates/ios.j2
+empty config_context: {}
+2023-12-12 11:46:39, INFO     | render_engine.py:347   | MainThread: device_b rendered file written: ./rendered_templates/device_b.txt

render_engine.py

@@ -5,13 +5,94 @@ from pathlib import Path
 import os
 import yaml
 import jinja2
+import toml
+import logging
 
-class FileChecks:
+## global logger
+logger = logging.getLogger('main')
+logger.setLevel(logging.INFO)
+console = logging.StreamHandler()
+file = logging.FileHandler('render_engine.log')
+logger.addHandler(console)
+logger.addHandler(file)
+formatter = logging.Formatter(
+  fmt="%(asctime)s, %(levelname)-8s | %(filename)-15s:%(lineno)-5s | %(threadName)-1s: %(message)s",
+  datefmt="%Y-%m-%d %H:%M:%S")
+console.setFormatter(formatter)
+file.setFormatter(formatter)
+
+## logic
+class ReadConfig():
+    'read or generate script config file'
+    def __init__(self, config_file):
+        # dirty way to write a text block
+        self.example_config = """\
+config_contexts_dir = './config_contexts'
+platform_templates_dir = './platform_templates'
+rendered_templates_dir = './rendered_templates'
+inventory_file = './inventory.yml'\
+"""
+        self.required_variables = ['config_contexts_dir', 'platform_templates_dir', 'rendered_templates_dir', 'inventory_file']
+        self.config_file = config_file
+        self.config_dict = self.__find_config(self.example_config, self.config_file)
+        self.__parseConfig(self.config_dict)
+
+    def __find_config(self, example_config, config_file):
+        if not Path(config_file).exists():
+            with open(config_file,'w') as config:
+                config.write(example_config)
+            logger.error(f'config missing, wrote example to {config_file} please modify and re-run')
+            sys.exit(0)
+        else:
+            try:
+                config_dict = toml.load(config_file)
+            except:
+                logger.error(f'invalid TOML format: {config_file}')
+                sys.exit(1)
+        return config_dict
+
+    def __parseConfig(self, config_dict):
+        # missing entries listed in self.required_variables
+        for i in self.required_variables:
+            try:
+                test = config_dict[i]
+            except:
+                logger.error(f'missing config entry: {i}')
+                sys.exit(1)
+        # empty entries in self.required_variables
+        for i in self.required_variables:
+            if not len(config_dict[i]):
+                logger.error(f'empty config entry: {i}')
+                sys.exit(1)
+        # missing files/directory in self.required_variables
+        missing = False
+        for i in self.required_variables:
+          if '_dir' in i:
+            directory = Path(config_dict[i])
+            if not directory.is_dir():
+              logger.error(f'missing directory: {i}= {config_dict[i]}')
+              missing = True
+          if '_file' in i:
+            file = Path(config_dict[i])
+            if not file.is_file():
+              print(f'missing file: {i} = {config_dict[i]}')
+              missing =True
+        if missing:
+          sys.exit(1)
+
+    def get_config(self):
+        return self.config_dict
+
+    def get_config_item(self, config_item):
+        if config_item in self.required_variables:
+          return self.config_dict[config_item]
+
+class InventoryFileChecks:
     'file check rules'
-    def __init__(self, config_contexts_dir, templates_dir, inventory_file):
+    def __init__(self, config_dict):
-      self.config_contexts_dir = config_contexts_dir
+      self.config_contexts_dir = config_dict['config_contexts_dir']
-      self.templates_dir = templates_dir
+      self.templates_dir = config_dict['platform_templates_dir']
-      self.inventory_file = inventory_file
+      self.inventory_file = config_dict['inventory_file']
       self.inventory_dict = self.__read_inventory_file(self.inventory_file)
       self.__check_context_files(self.config_contexts_dir)
       self.__check_platform_templates(self.inventory_dict, self.templates_dir)
@@ -22,13 +103,13 @@ class FileChecks:
             return yaml.safe_load(file)
         except Exception as ex:
           if type(ex).__name__ == 'ScannerError':
-            print(f'Error {inventory_file}: is not valid YAML')
+            logger.error(f'invalid YAML: {inventory_file}')
             sys.exit(1)
           elif type(ex).__name__ ==  'FileNotFoundError':
-            print(f'Error {inventory_file}: {ex.args[1]}')
+            logger.error(f'missing inventory file: {inventory_file}')
             sys.exit(1)
           else:
-            print(f'Error: {inventory_file}')
+            logger.error(f'unable to load inventory file: {inventory_file}')
             sys.exit(1)
 
     def __check_context_files(self, config_contexts_dir):
@@ -39,11 +120,11 @@ class FileChecks:
         file_names = [context_files[i]['file'] for i in context_files.keys()]
         duplicate_files = list(set([i for i in file_names if file_names.count(i) > 1]))
         if len(duplicate_files) >0:
-          print('==== to maintain readability there no logic to handle duplicate file names under the config_contexts directory')
+          logger.error(f'to maintain readability there no logic to handle duplicate file names under the config_contexts directory')
           for i in duplicate_files:
              for j in context_files.keys():
                 if context_files[j]['file'] == i:
-                   print(f"duplicate file name {context_files[j]['file']} @ {j}")
+                   logger.error(f"duplicate file name {context_files[j]['file']} @ {j}")
           sys.exit(1)
 
     def __check_platform_templates(self, inventory_dict, templates_dir):
@@ -53,14 +134,16 @@ class FileChecks:
           if not template.is_file():
             target_device_list = [k for k, v in inventory_dict.items() if v['platform'] == i]
             target_devices = '\n'.join(target_device_list)
-            print(f'==== missing template {template}, nothing will be rendered for devices:\n{target_devices}')
+            logger.error(f'missing template {template}, nothing will be rendered for devices:\n{target_devices}')
 
-class LoadInventory(FileChecks):
+class LoadInventory(InventoryFileChecks):
     'load inventory'
-    def __init__(self, config_contexts_dir, templates_dir, inventory_file):
+    def __init__(self, config_dict):
-        FileChecks.__init__(self, config_contexts_dir, templates_dir, inventory_file)
+        self.config_dict = config_dict
+        InventoryFileChecks.__init__(self, self.config_dict)
         self.all_config_contexts, self.devices = self.__load_contexts(self.inventory_dict)
-        #print(yaml.dump(self.all_config_contexts))
+        ## debug
+        # print(yaml.dump(self.all_config_contexts))
 
     def __remove_meta(self, data):
       try:
@@ -83,6 +166,7 @@ class LoadInventory(FileChecks):
               contexts_path.add(f'{self.config_contexts_dir}/{k}/{i}.yml')
           else:
             contexts_path.add(f'{self.config_contexts_dir}/{k}/{v}.yml')
+      ## debug
       # print(devices_path)
       # print(contexts_path)
 
@@ -99,6 +183,7 @@ class LoadInventory(FileChecks):
             all_config_contexts.update({context: {'weight': data['_metadata']['weight'], 'content': self.__remove_meta(data)}})
         except:
           pass
+      ## debug
       # print(all_config_contexts)
 
       # load device contexts
@@ -152,12 +237,16 @@ class LoadInventory(FileChecks):
           device_contexts.update({v: self.all_config_contexts[v]})
       return device_contexts
 
+    def get_rendered_templates_path(self):
+      return self.config_dict['rendered_templates_dir']
+
 class BuildDeviceContext:
     'build config_context for a device'
     def __init__(self, LoadInventory_instance, device):
       self.device = device
       self.contexts = LoadInventory_instance.get_device_contexts(self.device)
       self.platform = LoadInventory_instance.get_device_platform(self.device)
+      ## debug
       # print(self.contexts)
       # print(self.platform)
       self.config_context = self.__config_context(self.device, self.contexts)
@@ -172,6 +261,7 @@ class BuildDeviceContext:
           for k in contexts.keys():
             if contexts[k]['weight'] == i:
               equal_weight[i].append(k)
+        ## debug
         # print(f'equal weight contexts: {equal_weight}')
   
         # list all keys in equally weighted contexts
@@ -181,6 +271,7 @@ class BuildDeviceContext:
             for k in contexts[i]['content'].keys():
               keys_list.append(k)
         keys_list = list(set(keys_list))
+        ## debug
         # print(f'all keys in equal weight contexts: {keys_list}')
 
         # check each equally weighted context for clashing keys
@@ -194,10 +285,10 @@ class BuildDeviceContext:
             if len(key_occurance) >1:
               equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}')
         if len(equal_weight_key_clash_errors) >0:
-          error_log = f'==== device config_context render issue: {device}\n'
+          error_log = f'device config_context render issue: {device}'
           for i in equal_weight_key_clash_errors:
             error_log = error_log + f'\n{i}\n'
-          print(error_log)
+          logger.error(error_log)
           return {}
 
         # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting top level keys where they exist (the last context being the lowest weight and highest precedence)
@@ -206,6 +297,7 @@ class BuildDeviceContext:
         for k, v in contexts.items():
           config_context = {**config_context, **v['content']}
         config_context = dict(sorted(config_context.items()))
+        ## debug
         # print(yaml.dump(config_context))
         return config_context
 
@@ -220,13 +312,13 @@ class RenderDeviceConfig(BuildDeviceContext):
     def __init__(self, LoadInventory_instance, device):
         BuildDeviceContext.__init__(self, LoadInventory_instance, device)
         self.device_template = LoadInventory_instance.get_device_template(self.device)
+        self.rendered_template_path = LoadInventory_instance.get_rendered_templates_path()
         self.rendered_template = self.__rendered_template(self.device, self.config_context, self.device_template)
 
     def __rendered_template(self, device, config_context, device_template):
         if len(config_context) >0:
           template_directory = os.path.dirname(device_template)
           template_file = os.path.basename(device_template)
-          # rendered_template_file = f'{device}.txt'
           environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_directory))
           environment.trim_blocks = True
           environment.lstrip_blocks = True
@@ -240,51 +332,58 @@ class RenderDeviceConfig(BuildDeviceContext):
           # rendered_template_content = template.render(config_context)
           return rendered_template_content
         else:
-          print(f'==== device template render issue: {device}\n\n{os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context {config_context} (clashing top level keys in config_contexts)\n')
+          logger.error(f'device template render issue: {device}\ntemplate: {os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context: {config_context}')
           return
 
     def print_rendered_template(self):
         if self.rendered_template is not None:
-          print(f'==== {self.device} file content:\n\n{self.rendered_template}\n')
+          logger.info(f'{self.device} rendered file content:\n\n{self.rendered_template}\n')
 
     def write_rendered_template(self):
         if self.rendered_template is not None:
-          # this needs more logic to pass another parameter for dest dir
+          rendered_template_file = f'{self.rendered_template_path}/{self.device}.txt'
-          rendered_template_file = f'{self.device}.txt'
           with open(rendered_template_file, mode="w", encoding="utf-8") as message:
               message.write(self.rendered_template)
-              print(f'==== {self.device} file written: {rendered_template_file}')
+              logger.info(f'{self.device} rendered file written: {rendered_template_file}')
 
+## start
 def main():
-  config_contexts_dir = './config_contexts'
+
-  templates_dir = './templates'
+  # config validation
-  inventory_file = "./inventory.yml"
+  config_file = 'config.toml'
-  inventory = LoadInventory(config_contexts_dir, templates_dir, inventory_file)
+  config_obj = ReadConfig(config_file)
+
+  # load inventory
+  inventory = LoadInventory(config_obj.get_config())
+  ## debug
   # print(inventory.get_devices())
   # print(yaml.dump(inventory.get_contexts()))
 
+  # render templates
   config_contexts_list = []
   for i in inventory.get_devices():
     context_obj = RenderDeviceConfig(inventory, i)
     if len(context_obj.get_config_context()) >0:
+      ## debug
       # print(f'device:\n{context_obj.get_device_name()}\n')
       # print(f'config_context:\n{yaml.dump(context_obj.get_config_context())}')
       config_contexts_list.append(context_obj)
     else:
       del context_obj
 
+  # write rendered configs
   for i in config_contexts_list:
-    i.print_rendered_template()
     i.write_rendered_template()
+    ## debug
+    # i.print_rendered_template()
 
 if __name__ == "__main__":
     main()
 
-# turn this into a tool?
+
-#   - wrap up into parameterized script, try with a real template
+# parameters for:
-# add comment block to each renderd config such as rendered date (maybe list templates in use?)
+# - non standard config file location
-# needs a config file to point to contexts/folders, use toml top allow for comments
+# - show me all devices in inventory
-#   - feedback, the templates directory is confusing, templates/platforms
+# - target specific device for view or render
-# needs a rendered artefact directory parameter somewhere, maybe this should be a write_rendered_template parameter to allow it to be placed in another git repo
+# - view/render
-# needs proper logging
+# want a comment block or maybe for now just add a date/time in the file name - do this for now - this is going to be a git commit thing is there any point?
-# needs a mode to create example files/dirs

rendered_templates/device_b.txt

@@ -0,0 +1,78 @@
+**** start of template ****
+
+0) display the config_context:
+
+   top level context item:
+
+   {'best_unicorn': ['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love'], 'cbeebies_schedule': '18/11/2023', 'device': 'device_b', 'gabbys_doll_house': 1, 'harry_potter': 1, 'octonaughts': 1, 'region': 'europe', 'site': 'dublin'}
+
+   selecting the best unicorn with config_context['best_unicorn'][0]:
+
+   sugar socks
+
+   accessing a key pair injected into the jinja environment global variables:
+   (in this case the config_context printed out in human readable yaml format)
+
+   best_unicorn:
+   - sugar socks
+   - star light
+   - sparkle
+   - twinkle
+   - glitter love
+   cbeebies_schedule: 18/11/2023
+   device: device_b
+   gabbys_doll_house: 1
+   harry_potter: 1
+   octonaughts: 1
+   region: europe
+   site: dublin
+
+
+   loop through the config_context variable in jinja:
+
+   best_unicorn:['sugar socks', 'star light', 'sparkle', 'twinkle', 'glitter love']
+   cbeebies_schedule:18/11/2023
+   device:device_b
+   gabbys_doll_house:1
+   harry_potter:1
+   octonaughts:1
+   region:europe
+   site:dublin
+
+1) source template ./device/device_b.yml
+
+2) source template ./device/region/europe.yml
+
+3) source template ./device/site/dublin.yml
+
+4) source template ./device/role/<files>.yml
+   these contexts happens to be a list in the inventory
+   files under this directory happen to have the same weight(they dont have to)
+   thus these files do not have the same keys as they would clash
+
+   gabbys_doll_house = 1
+
+   octonaughts = 1
+
+   notice best_unicorn is in the all.yml context and router.yml context
+   router.yml has a higher precedence(lower weight value)
+
+   - sugar socks
+   - star light
+   - sparkle
+   - twinkle
+   - glitter love
+
+   harry_potter = 1
+   
+   notice cbeebies_schedule is in the all.yml context and vpn.yml context
+   all.yml context is weighted last/-1, any other context will overwrite it
+   vpn.yml has a higher precedence(lower weight value)
+
+   cbeebies_schedule = 18/11/2023
+
+4) source template ./device/role/all.yml
+   nothing is rendered using this context, all variables are overwritten by contexts with higher precedence
+   this would be a good place to put service accounts in ACLs or legal disclaimers
+
+**** end of template ****