gc_integration_templater/render_engine.py

#!/usr/bin/env python

import sys
from pathlib import Path
import os
import yaml
import jinja2
import toml
import logging

## global logger
logger = logging.getLogger('main')
logger.setLevel(logging.INFO)
console = logging.StreamHandler()
file = logging.FileHandler('render_engine.log')
logger.addHandler(console)
logger.addHandler(file)
formatter = logging.Formatter(
  fmt="%(asctime)s, %(levelname)-8s | %(filename)-15s:%(lineno)-5s | %(threadName)-1s: %(message)s",
  datefmt="%Y-%m-%d %H:%M:%S")
console.setFormatter(formatter)
file.setFormatter(formatter)

## logic
class ReadConfig():
    'read or generate script config file'
    def __init__(self, config_file):
        # dirty way to write a text block
        self.example_config = """\
config_contexts_dir = './config_contexts'
platform_templates_dir = './platform_templates'
rendered_templates_dir = './rendered_templates'
inventory_file = './inventory.yml'\
"""
        self.required_variables = ['config_contexts_dir', 'platform_templates_dir', 'rendered_templates_dir', 'inventory_file']
        self.config_file = config_file
        self.config_dict = self.__find_config(self.example_config, self.config_file)
        self.__parseConfig(self.config_dict)

    def __find_config(self, example_config, config_file):
        if not Path(config_file).exists():
            with open(config_file,'w') as config:
                config.write(example_config)
            logger.error(f'config missing, wrote example to {config_file} please modify and re-run')
            sys.exit(0)
        else:
            try:
                config_dict = toml.load(config_file)
            except:
                logger.error(f'invalid TOML format: {config_file}')
                sys.exit(1)
        return config_dict

    def __parseConfig(self, config_dict):
        # missing entries listed in self.required_variables
        for i in self.required_variables:
            try:
                test = config_dict[i]
            except:
                logger.error(f'missing config entry: {i}')
                sys.exit(1)
        # empty entries in self.required_variables
        for i in self.required_variables:
            if not len(config_dict[i]):
                logger.error(f'empty config entry: {i}')
                sys.exit(1)
        # missing files/directory in self.required_variables
        missing = False
        for i in self.required_variables:
          if '_dir' in i:
            directory = Path(config_dict[i])
            if not directory.is_dir():
              logger.error(f'missing directory: {i}= {config_dict[i]}')
              missing = True
          if '_file' in i:
            file = Path(config_dict[i])
            if not file.is_file():
              print(f'missing file: {i} = {config_dict[i]}')
              missing =True
        if missing:
          sys.exit(1)

    def get_config(self):
        return self.config_dict

    def get_config_item(self, config_item):
        if config_item in self.required_variables:
          return self.config_dict[config_item]

class InventoryFileChecks:
    'file check rules'
    def __init__(self, config_dict):
      self.config_contexts_dir = config_dict['config_contexts_dir']
      self.templates_dir = config_dict['platform_templates_dir']
      self.inventory_file = config_dict['inventory_file']
      self.inventory_dict = self.__read_inventory_file(self.inventory_file)
      self.__check_context_files(self.config_contexts_dir)
      self.__check_platform_templates(self.inventory_dict, self.templates_dir)

    def __read_inventory_file(self, inventory_file):
        try:
          with open(inventory_file, "r") as file:
            return yaml.safe_load(file)
        except Exception as ex:
          if type(ex).__name__ == 'ScannerError':
            logger.error(f'invalid YAML: {inventory_file}')
            sys.exit(1)
          elif type(ex).__name__ ==  'FileNotFoundError':
            logger.error(f'missing inventory file: {inventory_file}')
            sys.exit(1)
          else:
            logger.error(f'unable to load inventory file: {inventory_file}')
            sys.exit(1)

    def __check_context_files(self, config_contexts_dir):
        context_files = {}
        # check for contexts directory - need an all file, dont need a device file
        for path in Path(config_contexts_dir).rglob('*.yml'):
          context_files.update({str(path): { 'file': path.name,'path': str(path.parent)}})
        file_names = [context_files[i]['file'] for i in context_files.keys()]
        duplicate_files = list(set([i for i in file_names if file_names.count(i) > 1]))
        if len(duplicate_files) >0:
          logger.error(f'to maintain readability there no logic to handle duplicate file names under the config_contexts directory')
          for i in duplicate_files:
             for j in context_files.keys():
                if context_files[j]['file'] == i:
                   logger.error(f"duplicate file name {context_files[j]['file']} @ {j}")
          sys.exit(1)

    def __check_platform_templates(self, inventory_dict, templates_dir):
        platforms = set([inventory_dict[i]['platform'] for i in inventory_dict.keys()])
        for i in platforms:
          template = Path(f'{templates_dir}/{i}.j2')
          if not template.is_file():
            target_device_list = [k for k, v in inventory_dict.items() if v['platform'] == i]
            target_devices = '\n'.join(target_device_list)
            logger.error(f'missing template {template}, nothing will be rendered for devices:\n{target_devices}')

class LoadInventory(InventoryFileChecks):
    'load inventory'
    def __init__(self, config_dict):
        self.config_dict = config_dict
        InventoryFileChecks.__init__(self, self.config_dict)
        self.all_config_contexts, self.devices = self.__load_contexts(self.inventory_dict)
        ## debug
        # print(yaml.dump(self.all_config_contexts))

    def __remove_meta(self, data):
      try:
        del data['_metadata']
      except KeyError:
        pass
      return data

    def __load_contexts(self, inventory_dict):
      # get path of all device and context files
      devices_path = set()
      contexts_path = set()
      devices = []
      for i in inventory_dict.keys():
        devices_path.add(f'{self.config_contexts_dir}/device/{i}.yml')
        devices.append(i)
        for k, v in inventory_dict[i]['config_contexts'].items():      
          if isinstance(v, list):
            for i in v:
              contexts_path.add(f'{self.config_contexts_dir}/{k}/{i}.yml')
          else:
            contexts_path.add(f'{self.config_contexts_dir}/{k}/{v}.yml')
      ## debug
      # print(devices_path)
      # print(contexts_path)

      # load config contexts
      all_config_contexts = {}
      for i in contexts_path:
        try:
          with open(i, 'r') as file:
            data = yaml.safe_load(file)
          if data['_metadata']['weight'] and data['_metadata']['is_active']:
            # build the context_name from the file parent directory
            # context naming could be sourced from the metadata dict but the potential for duplicate filenames under config_contexts subdirectories is to be discouraged
            context = os.path.splitext(os.path.basename(i))[0]
            all_config_contexts.update({context: {'weight': data['_metadata']['weight'], 'content': self.__remove_meta(data)}})
        except:
          pass
      ## debug
      # print(all_config_contexts)

      # load device contexts
      for i in devices_path:
        try:
          with open(i, 'r') as file:
            data = yaml.safe_load(file)
          context = os.path.splitext(os.path.basename(i))[0]
          all_config_contexts.update({context: {'weight': 0, 'content': self.__remove_meta(data)}})
        except:
          pass

      # load all contexts
      all_weight = sorted([all_config_contexts[k]['weight'] for k, v in all_config_contexts.items()], reverse=True)
      max_weight = all_weight[0]+1
      try:
        with open(f'{self.config_contexts_dir}/all.yml', 'r') as file:
          data = yaml.safe_load(file)
          if data['_metadata']['is_active']:
            all_config_contexts.update({'all': {'weight': max_weight, 'content': self.__remove_meta(data)}})
      except:
        pass

      return all_config_contexts, devices

    def get_contexts(self):
      return self.all_config_contexts

    def get_devices(self):
      return self.devices

    def get_device_platform(self, device):
      return self.inventory_dict[device]['platform']

    def get_device_template(self, device):
      platform = self.get_device_platform(device)
      return f'{self.templates_dir}/{platform}.j2'

    def get_device_contexts(self, device):
      device_contexts = {}
      all_device_contexts = ['all', device]
      for i in all_device_contexts:
        if i in self.all_config_contexts:
          device_contexts.update({i: self.all_config_contexts[i]})
      for k, v in self.inventory_dict[device]['config_contexts'].items():  
        if isinstance(v, list):
          for i in v:
            if i in self.all_config_contexts:
              device_contexts.update({i: self.all_config_contexts[i]})
        elif v in self.all_config_contexts:
          device_contexts.update({v: self.all_config_contexts[v]})
      return device_contexts

    def get_rendered_templates_path(self):
      return self.config_dict['rendered_templates_dir']

class BuildDeviceContext:
    'build config_context for a device'
    def __init__(self, LoadInventory_instance, device):
      self.device = device
      self.contexts = LoadInventory_instance.get_device_contexts(self.device)
      self.platform = LoadInventory_instance.get_device_platform(self.device)
      ## debug
      # print(self.contexts)
      # print(self.platform)
      self.config_context = self.__config_context(self.device, self.contexts)

    def __config_context(self, device, contexts):
        # check for equally weighted clashing keys
        all_weight = sorted([contexts[k]['weight'] for k, v in contexts.items()], reverse=True)
        dupe_weight = list(set([x for x in all_weight if all_weight.count(x) > 1]))
        equal_weight = {}
        for i in dupe_weight:
          equal_weight.update({i: []})
          for k in contexts.keys():
            if contexts[k]['weight'] == i:
              equal_weight[i].append(k)
        ## debug
        # print(f'equal weight contexts: {equal_weight}')
  
        # list all keys in equally weighted contexts
        keys_list = []
        for k in equal_weight.keys():
          for i in equal_weight[k]:
            for k in contexts[i]['content'].keys():
              keys_list.append(k)
        keys_list = list(set(keys_list))
        ## debug
        # print(f'all keys in equal weight contexts: {keys_list}')

        # check each equally weighted context for clashing keys
        equal_weight_key_clash_errors = []
        for w in equal_weight:
          for k in keys_list:
            key_occurance = []
            for i in equal_weight[w]:
              if k in contexts[i]['content']:
                key_occurance.append(i)
            if len(key_occurance) >1:
              equal_weight_key_clash_errors.append(f'clashing key found in equally weighted contexts: \nkey: {k} \ncontexts: {key_occurance}')
        if len(equal_weight_key_clash_errors) >0:
          error_log = f'device config_context render issue: {device}'
          for i in equal_weight_key_clash_errors:
            error_log = error_log + f'\n{i}\n'
          logger.error(error_log)
          return {}

        # reorder all_config_contexts by weight, update the config_context with each context in order of precedence, overwriting top level keys where they exist (the last context being the lowest weight and highest precedence)
        config_context = {}
        contexts = {k: v for k, v in sorted(contexts.items(), key=lambda x: x[1]['weight'], reverse=True)}
        for k, v in contexts.items():
          config_context = {**config_context, **v['content']}
        config_context = dict(sorted(config_context.items()))
        ## debug
        # print(yaml.dump(config_context))
        return config_context

    def get_config_context(self):
      return self.config_context

    def get_device_name(self):
      return self.device

class RenderDeviceConfig(BuildDeviceContext):
    'build config_context for a device'
    def __init__(self, LoadInventory_instance, device):
        BuildDeviceContext.__init__(self, LoadInventory_instance, device)
        self.device_template = LoadInventory_instance.get_device_template(self.device)
        self.rendered_template_path = LoadInventory_instance.get_rendered_templates_path()
        self.rendered_template = self.__rendered_template(self.device, self.config_context, self.device_template)

    def __rendered_template(self, device, config_context, device_template):
        if len(config_context) >0:
          template_directory = os.path.dirname(device_template)
          template_file = os.path.basename(device_template)
          environment = jinja2.Environment(loader=jinja2.FileSystemLoader(template_directory))
          environment.trim_blocks = True
          environment.lstrip_blocks = True
          # add config_context dict to a global in the rendering environment, access variables like golden config
          environment.globals['config_context'] = config_context
          # pretty_config_context is for illustration only, non functional variable as keys cannot be accessed
          environment.globals['pretty_config_context'] = yaml.dump(config_context, default_flow_style=0)
          template = environment.get_template(template_file)
          # render without specifying variables or a dict as input, instead use environment.globals['config_context']
          rendered_template_content = template.render()
          # rendered_template_content = template.render(config_context)
          return rendered_template_content
        else:
          logger.error(f'device template render issue: {device}\ntemplate: {os.path.dirname(device_template)}/{os.path.basename(device_template)}\nempty config_context: {config_context}')
          return

    def print_rendered_template(self):
        if self.rendered_template is not None:
          logger.info(f'{self.device} rendered file content:\n\n{self.rendered_template}\n')

    def write_rendered_template(self):
        if self.rendered_template is not None:
          rendered_template_file = f'{self.rendered_template_path}/{self.device}.txt'
          with open(rendered_template_file, mode="w", encoding="utf-8") as message:
              message.write(self.rendered_template)
              logger.info(f'{self.device} rendered file written: {rendered_template_file}')

## start
def main():

  # config validation
  config_file = 'config.toml'
  config_obj = ReadConfig(config_file)

  # load inventory
  inventory = LoadInventory(config_obj.get_config())
  ## debug
  # print(inventory.get_devices())
  # print(yaml.dump(inventory.get_contexts()))

  # render templates
  config_contexts_list = []
  for i in inventory.get_devices():
    context_obj = RenderDeviceConfig(inventory, i)
    if len(context_obj.get_config_context()) >0:
      ## debug
      # print(f'device:\n{context_obj.get_device_name()}\n')
      # print(f'config_context:\n{yaml.dump(context_obj.get_config_context())}')
      config_contexts_list.append(context_obj)
    else:
      del context_obj

  # write rendered configs
  for i in config_contexts_list:
    i.write_rendered_template()
    ## debug
    # i.print_rendered_template()

if __name__ == "__main__":
    main()


# parameters for:
# - non standard config file location
# - show me all devices in inventory
# - target specific device for view or render
# - view/render
# want a comment block or maybe for now just add a date/time in the file name - do this for now - this is going to be a git commit thing is there any point?