gc_integration_templater/render_engine.py

#!/usr/bin/env python

import sys
from pathlib import Path
import os
import yaml
import jinja2

class FileChecks:
    'file check rules'
    def __init__(self, config_contexts_dir, templates_dir, inventory_file):
      self.config_contexts_dir = config_contexts_dir
      self.templates_dir = templates_dir
      self.inventory_file = inventory_file
      self.inventory_dict = self.__read_inventory_file(self.inventory_file)
      self.__check_context_files(self.config_contexts_dir)
      self.__check_platform_templates(self.inventory_dict, self.templates_dir)

    def __read_inventory_file(self, inventory_file):
        try:
          with open(inventory_file, "r") as file:
            return yaml.safe_load(file)
        except Exception as ex:
          if type(ex).__name__ == 'ScannerError':
            print(f'Error {inventory_file}: is not valid YAML')
            sys.exit(1)
          elif type(ex).__name__ ==  'FileNotFoundError':
            print(f'Error {inventory_file}: {ex.args[1]}')
            sys.exit(1)
          else:
            print(f'Error: {inventory_file}')
            sys.exit(1)

    def __check_context_files(self, config_contexts_dir):
        context_files = {}
        # check for contexts directory - need an all file, dont need a device file
        for path in Path(config_contexts_dir).rglob('*.yml'):
          context_files.update({str(path): { 'file': path.name,'path': str(path.parent)}})
        file_names = [context_files[i]['file'] for i in context_files.keys()]
        duplicate_files = list(set([i for i in file_names if file_names.count(i) > 1]))
        if len(duplicate_files) >0:
          print('==== to maintain readability there no logic to handle duplicate file names under the config_contexts directory')
          for i in duplicate_files:
             for j in context_files.keys():
                if context_files[j]['file'] == i:
                   print(f"duplicate file name {context_files[j]['file']} @ {j}")
          sys.exit(1)

    def __check_platform_templates(self, inventory_dict, templates_dir):
        platforms = set([inventory_dict[i]['platform'] for i in inventory_dict.keys()])
        for i in platforms:
          template = Path(f'{templates_dir}/{i}.j2')
          if not template.is_file():
            target_device_list = [k for k, v in inventory_dict.items() if v['platform'] == i]
            target_devices = '\n'.join(target_device_list)
            print(f'==== missing template {template}, nothing will be rendered for devices:\n{target_devices}')

class LoadInventory(FileChecks):
    'load inventory'
    def __init__(self, config_contexts_dir, templates_dir, inventory_file):
        FileChecks.__init__(self, config_contexts_dir, templates_dir, inventory_file)
        self.all_config_contexts, self.devices = self.__load_contexts(self.inventory_dict)
        #print(yaml.dump(self.all_config_contexts))

    def __remove_meta(self, data):
      try:
        del data['_metadata']
      except KeyError:
        pass
      return data

    def __load_contexts(self, inventory_dict):
      # get path of all device and context files
      devices_path = set()
      contexts_path = set()
      devices = []
      for i in inventory_dict.keys():
        devices_path.add(f'{self.config_contexts_dir}/device/{i}.yml')
        devices.append(i)
        for k, v in inventory_dict[i]['config_contexts'].items():      
          if isinstance(v, list):
            for i in v:
              contexts_path.add(f'{self.config_contexts_dir}/{k}/{i}.yml')
          else:
            contexts_path.add(f'{self.config_contexts_dir}/{k}/{v}.yml')
      # print(devices_path)
      # print(contexts_path)

      # load config contexts
      all_config_contexts = {}
      for i in contexts_path:
        try:
          with open(i, 'r') as file:
            data = yaml.safe_load(file)
          if data['_metadata']['weight'] and data['_metadata']['is_active']:
            # build the context_name from the file parent directory
            # context naming could be sourced from the metadata dict but the potential for duplicate filenames under config_contexts subdirectories is to be discouraged
            context = os.path.splitext(os.path.basename(i))[0]
            all_config_contexts.update({context: {'weight': data['_metadata']['weight'], 'content': self.__remove_meta(data)}})
        except:
          pass
      # print(all_config_contexts)

      # load device contexts
      for i in devices_path:
        try:
          with open(i, 'r') as file:
            data = yaml.safe_load(file)
          context = os.path.splitext(os.path.basename(i))[0]
          all_config_contexts.update({context: {'weight': 0, 'content': self.__remove_meta(data)}})
        except:
          pass

      # load all contexts
      all_weight = sorted([all_config_contexts[k]['weight'] for k, v in all_config_contexts.items()], reverse=True)
      max_weight = all_weight[0]+1
      try:
        with open(f'{self.config_contexts_dir}/all.yml', 'r') as file:
          data = yaml.safe_load(file)
          if data['_metadata']['is_active']:
            all_config_contexts.update({'all': {'weight': max_weight, 'content': self.__remove_meta(data)}})
      except:
        pass

      return all_config_contexts, devices

    def get_contexts(self):
      return self.all_config_contexts

    def get_devices(self):
      return self.devices

    def get_device_platform(self, device):
      return self.inventory_dict[device]['platform']

    def get_device_template(self, device):
      platform = self.get_device_platform(device)
      return f'{self.templates_dir}/{platform}.j2'

    def get_device_contexts(self, device):
      device_contexts = {}
      all_device_contexts = ['all', device]
      for i in all_device_contexts:
        if i in self.all_config_contexts:
          device_contexts.update({i: self.all_config_contexts[i]})
      for k, v in self.inventory_dict[device]['config_contexts'].items():  
        if isinstance(v, list):
          for i in v:
            if i in self.all_config_contexts:
              device_contexts.update({i: self.all_config_contexts[i]})
        elif v in self.all_config_contexts:
          device_contexts.update({v: self.all_config_contexts[v]})
      return device_contexts

class BuildDeviceContext:
    'build config_context for a device'
    def __init__(self, LoadInventory_instance, device):
      self.device = device
      self.contexts = LoadInventory_instance.get_device_contexts(self.device)
      self.platform = LoadInventory_instance.get_device_platform(self.device)
      # print(self.contexts)
      # print(self.platform)
      self.config_context = self.__config_context(self.device, self.contexts)

    def __config_context(self, device, contexts):
        # check for equally weighted clashing keys
        all_weight = sorted([contexts[k]['weight'] for k, v in contexts.items()], reverse=True)
        dupe_weight = list(set([x for x in all_weight if all_weight.count(x) > 1]))
        equal_weight = {}
        for i in dupe_weight:
          equal_weight.update({i: []})
          for k in contexts.keys():
            if contexts[k]['weight'] == i:
              equal_weight[i].append(k)
        # print(f'equal weight contexts: {equal_weight}')
  
        # list all keys in equally weighted contexts
        keys_list = []
        for k in equal_weight.keys():
          for i in equal_weight[k]:
            for k in contexts[i]['content'].keys():
              keys_list.append(k)
        keys_list = list(set(keys_list))
        # print(f'all keys in equal weight contexts: {keys_list}')

        # check each equally weighted context for clashing keys
        equal_weight_key_clash_errors = []
        for w in equal_weight:
          for k in keys_list:
            key_occurance = []
            for i in equal_weight[w]:
              if k in contexts[i]['content']:
                key_occurance.append(i)
            if len(key_occurance) >1:
              equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}')
        if len(equal_weight_key_clash_errors) >0:
          error_log = f'==== device config_context render issue: {device}\n'
          for i in equal_weight_key_clash_errors:
            error_log = error_log + f'\n{i}\n'
          print(error_log)
          return {}

        # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting top level keys where they exist (the last context being the lowest weight and highest precedence)
        config_context = {}
        contexts = {k: v for k, v in sorted(contexts.items(), key=lambda x: x[1]['weight'], reverse=True)}
        for k, v in contexts.items():
          config_context = {**config_context, **v['content']}
        config_context = dict(sorted(config_context.items()))
        # print(yaml.dump(config_context))
        return config_context

    def get_config_context(self):
      return self.config_context

    def get_device_name(self):
      return self.device

class RenderDeviceConfig(BuildDeviceContext):
    'build config_context for a device'
    def __init__(self, LoadInventory_instance, device):
        BuildDeviceContext.__init__(self, LoadInventory_instance, device)
        self.device_template = LoadInventory_instance.get_device_template(self.device)
        self.rendered_template = self.__rendered_template(self.device, self.config_context, self.device_template)

    def __rendered_template(self, device, config_context, device_template):
        if len(config_context) >0:
          template_directory = os.path.dirname(device_template)
          template_file = os.path.basename(device_template)
          # rendered_template_file = f'{device}.txt'
          environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_directory))
          environment.trim_blocks = True
          environment.lstrip_blocks = True
          # add config_context dict to a global in the rendering environment, access variables like golden config
          environment.globals['config_context'] = config_context
          # pretty_config_context is for illustration only, non functional variable as keys cannot be accessed
          environment.globals['pretty_config_context'] = yaml.dump(config_context, default_flow_style=0)
          template = environment.get_template(template_file)
          # render without specifying variables or a dict as input, instead use environment.globals['config_context']
          rendered_template_content = template.render()
          # rendered_template_content = template.render(config_context)
          return rendered_template_content
        else:
          print(f'==== device template render issue: {device}\n\n{os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context {config_context} (clashing top level keys in config_contexts)\n')
          return

    def print_rendered_template(self):
        if self.rendered_template is not None:
          print(f'==== {self.device} file content:\n\n{self.rendered_template}\n')

    def write_rendered_template(self):
        if self.rendered_template is not None:
          # this needs more logic to pass another parameter for dest dir
          rendered_template_file = f'{self.device}.txt'
          with open(rendered_template_file, mode="w", encoding="utf-8") as message:
              message.write(self.rendered_template)
              print(f'==== {self.device} file written: {rendered_template_file}')

def main():
  config_contexts_dir = './config_contexts'
  templates_dir = './templates'
  inventory_file = "./inventory.yml"
  inventory = LoadInventory(config_contexts_dir, templates_dir, inventory_file)
  # print(inventory.get_devices())
  # print(yaml.dump(inventory.get_contexts()))

  config_contexts_list = []
  for i in inventory.get_devices():
    context_obj = RenderDeviceConfig(inventory, i)
    if len(context_obj.get_config_context()) >0:
      # print(f'device:\n{context_obj.get_device_name()}\n')
      # print(f'config_context:\n{yaml.dump(context_obj.get_config_context())}')
      config_contexts_list.append(context_obj)
    else:
      del context_obj

  for i in config_contexts_list:
    i.print_rendered_template()
    i.write_rendered_template()

if __name__ == "__main__":
    main()

# turn this into a tool?
#   - wrap up into parameterized script, try with a real template
# add comment block to each renderd config such as rendered date (maybe list templates in use?)
# needs a config file to point to contexts/folders, use toml top allow for comments
#   - feedback, the templates directory is confusing, templates/platforms
# needs a rendered artefact directory parameter somewhere, maybe this should be a write_rendered_template parameter to allow it to be placed in another git repo
# needs proper logging
# needs a mode to create example files/dirs