import config from functools import partial from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor from scrapli import Scrapli # from scrapli.driver import GenericDriver # from scrapli.driver.core import IOSXEDriver from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne, UpdateMany import re import json import logging from netaddr import valid_ipv4 def cisco_version(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] id = device_record['_id'] output = command_output['output'] ## log # print('run cisco_version') logger = logging.getLogger(device_name) logger.info('Run: cisco_version') ## parse and update if not output == 'error': if len(output.genie_parse_output()) >0: # print(f'genie parser method') logger.info(f'genie parser method') parsed = output.genie_parse_output() operating_system = parsed['version']['os'] image = parsed['version']['system_image'].split(':')[1].replace('/', '') version = parsed['version']['version'] chassis = parsed['version']['chassis'] serial = parsed['version']['chassis_sn'] record = {'os_flavour': operating_system, 'image': image, 'os_version': version, 'chassis': chassis, 'serial': serial} # print(record) # logger.info(record) filter = {'_id': id} result = collection.update_one(filter, {'$set': record}, upsert=True) logger.info(f'Database: matched_count {result.matched_count} modified_count {result.modified_count}') return result else: # print('scrape collection error') logger.error(f'Scrape collection error') #### scrape 1) # 'show isakmp sa detail' # {'c_id': '20681', 'local_ip': '10.227.184.157', 'p1_ivrf': 'none', 'peer_ip': '10.229.4.74', 'p1_dh_group': '14', 'p1_encr_algo': 'aes', 'p1_hash_algo': 'sha', 'p1_auth_type': 'psk', 'p1_status': 'ACTIVE'} def cisco_vpn_phase1(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_table = collection['temp'][device_name] # create/use-existing temp subcollection output = command_output['output'] # print(output.result) ## log # print('\ncisco_vpn_phase1') logger = logging.getLogger(device_name) logger.info('Run: cisco_vpn_phase1') def process_p1(p1_dict, idx): global peer_count global p1_records c_id = str(p1_dict['isakmp_stats']['IPv4'][idx]['c_id']) local_ip = p1_dict['isakmp_stats']['IPv4'][idx]['local_ip'] peer_ip = p1_dict['isakmp_stats']['IPv4'][idx]['remote_ip'] encr_algo = p1_dict['isakmp_stats']['IPv4'][idx]['encr_algo'] hash_algo = p1_dict['isakmp_stats']['IPv4'][idx]['hash_algo'] auth_type = p1_dict['isakmp_stats']['IPv4'][idx]['auth_type'] dh_group = str(p1_dict['isakmp_stats']['IPv4'][idx]['dh_group']) status = p1_dict['isakmp_stats']['IPv4'][idx]['status'] ivrf = p1_dict['isakmp_stats']['IPv4'][idx]['ivrf'] if 'ivrf' in p1_dict['isakmp_stats']['IPv4'][idx] else 'none' p1_record = {'c_id': c_id, 'local_ip': local_ip, 'p1_ivrf': ivrf, 'peer_ip': peer_ip, 'p1_dh_group': dh_group, 'p1_encr_algo': encr_algo, 'p1_hash_algo': hash_algo, 'p1_auth_type': auth_type, 'p1_status': status } # print(f'genie p1 {p1_records}') p1_records.append(p1_record) peer_count += 1 # print('phase1 processed %d\r'%peer_count, end="") # rolling count for single device mode ## parse and update if not output == 'error': p1_dict = output.genie_parse_output() # print(json.dumps(p1_dict, indent=4)) if len(p1_dict) >0: global peer_count peer_count = 0 global p1_records p1_records = [] p1_scrape_idx = [t for t in p1_dict['isakmp_stats']['IPv4']] partial_function = partial(process_p1, p1_dict) # with ThreadPoolExecutor(max_workers=1) as executor: # debug with ThreadPoolExecutor(max_workers=config.scrape_threads) as executor: executor.map(partial_function, p1_scrape_idx) # print(f'Phase1 processed {peer_count}') logger.info(f'Phase1 processed {peer_count}') # write to db if len(p1_records) >0: requests = [] for i in p1_records: record = i # requests.append(InsertOne(record)) filter = {'c_id': record['c_id']} requests.append(UpdateMany(filter, {'$set': record}, upsert=True)) result = device_table.bulk_write(requests) # print(result.bulk_api_result) # logger.info(result.bulk_api_result) # raw bulk api result with upserted ObjectId logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') return result else: # print('no phase1 tunnel records') logger.info('no phase1 tunnel records') else: # print('Scrape collection error') logger.info('Scrape collection error') #### scrape 2) # 'show crypto session remote {ip} detail' # {'local_ip': '10.225.112.42', 'local_port': '500', 'c_id': '11907', 'ipsec_flow': ['permit 47 host 10.225.112.42 host 10.227.36.18'], 'crypto_session_interface': 'Tunnel6', 'session_status': 'UP-ACTIVE', 'peer_ip': '10.227.36.18', 'peer_port': '500', 'p2_fvrf': 'none', 'peer_vpn_id': '10.227.36.18'} # correlate to scrape 1) with key 'c_id' def cisco_crypto_session(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_table = collection['temp'][device_name] # create/use-existing temp subcollection output = command_output['output'] # print(output.result) ## log # print('\ncisco_vpn_phase1') logger = logging.getLogger(device_name) logger.info('Run: cisco_crypto_session') def process_interface(interface): # debug with single thread # print('\n##########') # print(interface) global crypto_session_count global session_records interface_record_dict = {} all_sessions = "" all_sessions_found = False # breakout each interface entry containing the sessions for line in interface.split('\n'): if 'Interface: ' in line: crypto_session_interface = line.split(' ')[1] interface_record_dict.update({'crypto_session_interface': crypto_session_interface}) if 'Profile: ' in line: p1_profile = line.split(' ')[1] interface_record_dict.update({'p1_profile': p1_profile}) if 'Session status: ' in line: session_status = line.split('Session status: ')[1] interface_record_dict.update({'session_status': session_status}) if 'Peer: ' in line: peer_ip = line.split(' ')[1] peer_port = line.split(' ')[3] p2_fvrf = line.split(' ')[5].replace('(', '').replace(')', '') # p1_vrf = line.split(' ')[7] interface_record_dict.update({'peer_ip': peer_ip, 'peer_port': peer_port, 'p2_fvrf': p2_fvrf}) if 'Phase1_id: ' in line: peer_vpn_id = line.lstrip().split(' ')[1].replace('(', '').replace(')', '') interface_record_dict.update({'peer_vpn_id': peer_vpn_id}) # split all lines from 'IKEv1 SA: ' to end of scrape, capture all sessions if not all_sessions_found: if any(ike in line for ike in ['IKEv1 SA: ', 'IKE SA: ']): all_sessions += f'{line}\n' all_sessions_found = True elif 'Session ID: ' in line: pass else: all_sessions += f'{line}\n' # print(interface_record_dict) # breakout interface sessions, scrape from 'IKEv1 SA:' to 'IKEv1 SA:' # valid sessions: # P1 only with 'IKEv1 SA: ' # P1 + P2 with 'IKEv1 SA: ' and 'IPSEC FLOW: ' # P1 + (+)P1 + P2 with 'IKEv1 SA: ', 'IKEv1 SA: ' and 'IPSEC FLOW: ' # multiple preceeding 'IKEv1 SA:' with no IPSEC FLOW: means renegotiation/history, they have same key attributes differing only by differing c_id(connid) # see the below session scrape # # IKEv1 SA: local 10.224.6.130/4500 remote 213.26.160.84/4500 Active # Capabilities:DN connid:61862 lifetime:21:53:54 # # IKEv1 SA: local 10.224.6.130/4500 remote 213.26.160.84/4500 Active # Capabilities:DN connid:61863 lifetime:21:54:01 # # IKEv1 SA: local 10.224.6.130/4500 remote 213.26.160.84/4500 Active # Capabilities:DN connid:61864 lifetime:21:54:01 # IPSEC FLOW: permit 47 host 10.224.6.130 host 213.26.160.84 # Active SAs: 2, origin: crypto map # Inbound: #pkts dec'ed 979618 drop 0 life (KB/Sec) 4607998/208 # Outbound: #pkts enc'ed 986169 drop 0 life (KB/Sec) 4607998/208 # sessions = [] for line in all_sessions.split('\n'): if any(ike in line for ike in ['IKEv1 SA: ', 'IKE SA: ']): sessions.append(f'{line}\n') else: sessions[-1] += f'{line}\n' # print('## sessions ##') # for session in sessions: # print(f'\n%s' % session.split("\n")) # scrape each session attribute to keypairs # # by declaring this dict outside of the loop each session will be updating the same record (this negates the need for the previous step really) # as the IKE sessions share the same attributes this removes further deduplication later on session_record_dict = {} for session in sessions: # print(f'\n%s' % session.split("\n")) ipsec_flow =[] for line in session.split('\n'): # if 'IKEv1 SA: ' in line: # fails for older cisco if any(ike in line for ike in ['IKEv1 SA: ', 'IKE SA: ']): local_ip = line.lstrip().split(' ')[3].split('/')[0] local_port = line.lstrip().split(' ')[3].split('/')[1] session_record_dict.update({'local_ip': local_ip, 'local_port': local_port}) if 'connid:' in line: c_id = line.lstrip().split(' ')[1].split(':')[1] session_record_dict.update({'c_id': c_id}) if 'IPSEC FLOW: ' in line: acl = line.lstrip().split('FLOW: ')[1] ipsec_flow.append(acl) if len(ipsec_flow) >0: session_record_dict.update({'ipsec_flow': ipsec_flow}) # merge interface record component with session record session_record_dict.update(interface_record_dict) # print(session_record_dict) # check for required fields for complete/valid session record if all(m in session_record_dict for m in ['local_ip', 'peer_ip', 'c_id']): crypto_session_count += 1 # print('crypto session processed %d\r'%crypto_session_count, end="") session_records.append(session_record_dict) ## parse and update if not output == 'error': global crypto_session_count crypto_session_count = 0 global session_records session_records = [] # strip everything up to first 'Interface: ' scrape = "" tag_found = False for line in output.result.split('\n'): if not tag_found: if 'Interface: ' in line: scrape += f'{line}\n' tag_found = True else: scrape += f'{line}\n' # print(scrape) # split document into cryptomap entries interfaces = [] interfaces_found_count = 0 try: if len(scrape) >0: for line in scrape.split('\n'): if 'Interface: ' in line: interfaces.append(f'{line}\n') interfaces_found_count += 1 # print('lookup crypto session %d\r'%interfaces_found_count, end="") else: interfaces[-1] += f'{line.lstrip()}\n' # print(f'lookup crypto sessions {interfaces_found_count}') logger.info(f'lookup crypto sessions {interfaces_found_count}') except Exception as e: # print(f'Failed to process crypto session scrape: {e}') # print(scrape) logger.error(f'Failed to process crypto session scrape: {e}') pass # process crypto session interface scrapes partial_function = partial(process_interface) with ThreadPoolExecutor(max_workers=config.scrape_threads) as executor: executor.map(partial_function, interfaces) # print(f'crypto sessions processed {crypto_session_count}') logger.info(f'crypto sessions processed {crypto_session_count}') # write to db if len(session_records) >0: requests = [] for i in session_records: record = i filter = {'c_id': record['c_id']} requests.append(UpdateMany(filter, {'$set': record}, upsert=True)) result = device_table.bulk_write(requests) # print(result.bulk_api_result) # logger.info(result.bulk_api_result) logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') return result else: # print('Scrape collection error') logger.error('Scrape collection error') #### scrape 3) # 'show crypto ipsec sa' # {'p2_interface': 'Tunnel26', 'local_ip': '10.224.7.38', 'protected_vrf': 'FSD-PSD', 'peer_ip': '94.142.235.73', 'peer_port': '4500', 'pfs': 'N', 'p2_encr_algo': 'esp-3des', 'p2_hash_algo': 'esp-sha-hmac', 'crypto_map': 'Tunnel26-head-0', 'p2_status': 'ACTIVE'} # correlate to (scrape 1) + scrape 2)) with keys 'local_ip' 'peer_ip' 'peer_port' def cisco_vpn_phase2(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_table = collection['temp'][device_name] # create/use-existing temp subcollection output = command_output['output'] # print(output.result) ## log # print('\ncisco_vpn_phase2') logger = logging.getLogger(device_name) logger.info('Run: cisco_vpn_phase2') def process_vrfs(vrf): global ipsec_tunnel_processed_count global p2_records p2_record_dict = vrf.copy() p2_record_dict.pop('vrf') # get vrf vars for line in vrf['vrf'].split('\n'): if 'current_peer' in line: p2_record_dict.update({'peer_ip': line.lstrip(' ').split(' ')[1]}) p2_record_dict.update({'peer_port': line.lstrip(' ').split(' ')[3]}) if 'PFS' in line: p2_record_dict.update({'pfs': line.lstrip(' ').split(' ')[2].split(',')[0].upper()}) if 'transform' in line: transform = line.lstrip(' ').split('transform: ')[1].split(' ,')[0] p2_record_dict.update({'p2_encr_algo': transform.split(' ')[0]}) p2_record_dict.update({'p2_hash_algo': transform.split(' ')[1]}) del transform if 'crypto map: ' in line: p2_record_dict.update({'crypto_map': line.lstrip(' ').split('crypto map: ')[1]}) if 'Status' in line: p2_record_dict.update({'p2_status': line.split('Status: ')[1].split('(')[0]}) if 'protected vrf' in line: p2_record_dict.update({'protected_vrf': line.split('protected vrf: ')[1].replace('(', '').replace(')', '')}) # print(p2_record_dict) # check for required fields for complete/valid p2 record if all(include in p2_record_dict for include in ['local_ip', 'peer_ip', 'peer_port']): # print(p2_record_dict) p2_records.append(p2_record_dict) ipsec_tunnel_processed_count += 1 # print('processed ipsec sa %d\r'%ipsec_tunnel_processed_count, end="") ## parse and update if not output == 'error': global ipsec_tunnel_processed_count ipsec_tunnel_processed_count =0 global p2_records p2_records = [] # strip everything up to first 'interface: ' scrape = "" tag_found = False for line in output.result.split('\n'): if not tag_found: if 'interface: ' in line: scrape += f'{line}\n' tag_found = True else: scrape += f'{line}\n' # print(scrape) # split document into interface entries interfaces = [] interfaces_found_count = 0 if len(scrape) >0: for line in scrape.split('\n'): if 'interface: ' in line: interfaces.append(f'{line}\n') interfaces_found_count += 1 # print('lookup ipsec interface %d\r'%interfaces_found_count, end="") else: interfaces[-1] += f'{line.lstrip()}\n' # print(f'lookup ipsec interfaces {interfaces_found_count}') logger.info(f'lookup ipsec interfaces {interfaces_found_count}') # split interfaces into protected vrfs threadpool_items = [] for int in interfaces: int_record = {} protected_vrfs_found = False all_protected_vrfs = "" protected_vrfs = [] # get interface attributes for line in int.split('\n'): if 'interface: ' in line: int_record.update({'p2_interface': line.split(' ')[1]}) if 'local addr ' in line: int_record.update({'local_ip': line.split('addr ')[1]}) # split all lines from 'protected vrf: ' to end of scrape, capture all protected vrfs if not protected_vrfs_found: if 'protected vrf: ' in line: all_protected_vrfs += f'{line}\n' protected_vrfs_found = True else: all_protected_vrfs += f'{line}\n' # get all protected vrfs for line in all_protected_vrfs.split('\n'): if 'protected vrf: ' in line: protected_vrfs.append(f'{line.lstrip()}\n') else: protected_vrfs[-1] += f'{line.lstrip()}\n' # create threadpool items list for vrf in protected_vrfs: vrf_record = {'vrf': vrf} vrf_record.update(int_record) threadpool_items.append(vrf_record) # print(f'lookup protected vrf statements {len(threadpool_items)}') logger.info(f'lookup protected vrfs {len(threadpool_items)}') # process phase2 scrapes partial_function = partial(process_vrfs) with ThreadPoolExecutor(max_workers=config.scrape_threads) as executor: executor.map(partial_function, threadpool_items) # print(f'processed ipsec sa {ipsec_tunnel_processed_count}') logger.info(f'processed ipsec SA {ipsec_tunnel_processed_count}') # write to db if len(p2_records) >0: requests = [] for i in p2_records: record = i filter = {'local_ip': record['local_ip'], 'peer_ip': record['peer_ip'], 'peer_port': record['peer_port']} requests.append(UpdateMany(filter, {'$set': record}, upsert=True)) result = device_table.bulk_write(requests) # print(result.bulk_api_result) # logger.info(result.bulk_api_result) logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') return result else: # print('Scrape collection error') logger.error('Scrape collection error') #### scrape 4) # {'crypto_map': 'Tunnel6-head-0', 'peer_ip': '10.227.112.50', 'pfs': 'N', 'transform_sets': [{'name': 'TS-AES256-SHA', 'p2_encr_algo': 'esp-256-aes', 'p2_hash_algo': 'esp-sha-hmac'}, {'name': 'TS-3DES-SHA', 'p2_encr_algo': 'esp-3des', 'p2_hash_algo': 'esp-sha-hmac'}], 'crypto_map_interface': ['Tunnel6'], 'RRI_enabled': False, 'default_p2_3des': False} # correlate to (scrape 1) + scrape 2) + scrape 3)) with keys 'peer_ip' 'crypto_map' def cisco_crypto_map(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_table = collection['temp'][device_name] # create/use-existing temp subcollection output = command_output['output'] # print(output.result) ## log # print('\ncisco_crypto_map') logger = logging.getLogger(device_name) logger.info('Run: cisco_crypto_map') def process_cryptomaps(cryptomap): # debug with single thread # print('\n##########') # print(cryptomap) global crypto_map_count global cryptomap_records tfs_found = False int_found = False tfset = [] crypto_map_interface = [] cryptomap_record_dict = {} # scrape each cryptomap attribute to keypairs for line in cryptomap.split('\n'): if 'Crypto Map "' in line: # older variant of ios crypto_map = line.split(' ')[2].replace('"', '') cryptomap_record_dict.update({'crypto_map': crypto_map}) if 'Crypto Map IPv4 "' in line: # newer variant of ios crypto_map = line.split(' ')[3].replace('"', '') cryptomap_record_dict.update({'crypto_map': crypto_map}) if 'ISAKMP Profile: ' in line: p1_profile = line.split(' ')[2] cryptomap_record_dict.update({'p1_profile': p1_profile}) if 'Current peer: ' in line: peer_ip = line.split(' ')[2] cryptomap_record_dict.update({'peer_ip': peer_ip}) # RRI devices use dynamic crypto map templates, the name of the crypto map may not match the template name CM-BML-RRI != CDM-BML-RRI if 'dynamic (created from dynamic map ' in line: # dynamic (created from dynamic map CDM-BML-RRI/200) crypto_map_template = line.split('dynamic map ')[1].split('/')[0] cryptomap_record_dict.update({'crypto_map_template': crypto_map_template}) if 'PFS (Y/N): ' in line: pfs = line.split(' ')[2].upper() cryptomap_record_dict.update({'pfs': pfs}) if not tfs_found: if 'Transform sets=' in line: tfs_found = True pass elif ' } ,' in line: tfs_name = line.split(' ')[0].split(':')[0] tfs_encr_algo = line.replace(' ', ' ').split(' ')[2] tfs_hash_algo = line.replace(' ', ' ').split(' ')[3] tfset.append({'name': tfs_name, 'p2_encr_algo': tfs_encr_algo, 'p2_hash_algo': tfs_hash_algo}) else: tfs_found = False if 'Reverse Route Injection Enabled' in line: cryptomap_record_dict.update({'RRI_enabled': True}) if not int_found: if 'Interfaces using crypto map ' in line: int_found = True pass else: if len(line) >0: crypto_map_interface.append(line) # add possible list items to cryptomap record if len(tfset) >0: cryptomap_record_dict.update({'transform_sets': tfset}) if len(crypto_map_interface) >0: cryptomap_record_dict.update({'crypto_map_interface' : crypto_map_interface}) # catch absence of RRI if 'RRI_enabled' not in cryptomap_record_dict: cryptomap_record_dict.update({'RRI_enabled': False}) # # DISABLE - transform_sets are dynamicly ordered based on handshake in this command, not a source of truth # # ordered transform sets can be found from - show config / show crypto ipsec profile / show crypto dynamic-map # # # # determine if 1st/default P2 transform set is 3des # if 'transform_sets' in cryptomap_record_dict: # if '3des' in cryptomap_record_dict['transform_sets'][0]['p2_encr_algo'].lower(): # cryptomap_record_dict.update({'default_p2_3des': True}) # else: # cryptomap_record_dict.update({'default_p2_3des': False}) # # print(cryptomap_record_dict) # check for required fields for complete/valid cryptomap record (if the cryptomap has no peer_ip it has no use) if all(include in cryptomap_record_dict for include in ['peer_ip', 'crypto_map']): # print(cryptomap_record_dict) cryptomap_records.append(cryptomap_record_dict) crypto_map_count += 1 # print('cryptomaps processed %d\r'%crypto_map_count, end="") ## parse and update if not output == 'error': global crypto_map_count crypto_map_count = 0 global cryptomap_records cryptomap_records = [] # strip everything up to first 'Crypto Map IPv4 ' scrape = "" tag_found = False for line in output.result.split('\n'): if not tag_found: #if 'Crypto Map IPv4 ' in line: if 'Crypto Map ' in line: scrape += f'{line}\n' tag_found = True else: scrape += f'{line}\n' # print(scrape) # split document into cryptomap entries cryptomaps = [] crypto_map_found_count = 0 try: if len(scrape) >0: for line in scrape.split('\n'): # this can fail but only on a huge scrape which is hard to see - mep-shared-rri-agg09 # if 'Crypto Map IPv4 ' in line: # will not work on older cisco if 'Crypto Map ' in line: cryptomaps.append(f'{line}\n') crypto_map_found_count += 1 # print('lookup crypto maps %d\r'%crypto_map_found_count, end="") elif 'Crypto Map: ' in line: # these lines list the isakmp profile for the ipsec cryptomap profile, shorthand output that is not required pass else: cryptomaps[-1] += f'{line.lstrip()}\n' # print(f'lookup crypto maps {crypto_map_found_count}') logger.info(f'lookup crypto maps {crypto_map_found_count}') except Exception as e: # print(f'Failed to process crypto map scrape: {e}') # print(scrape) logger.error(f'Failed to process crypto map scrape: {e}') pass # process cryptomap scrapes partial_function = partial(process_cryptomaps) with ThreadPoolExecutor(max_workers=config.scrape_threads) as executor: executor.map(partial_function, cryptomaps) # print(f'crypto maps processed {crypto_map_count}') logger.info(f'crypto maps processed {crypto_map_count}') # write to db if len(cryptomap_records) >0: requests = [] for i in cryptomap_records: record = i filter = {'peer_ip': record['peer_ip'], 'crypto_map': record['crypto_map']} requests.append(UpdateMany(filter, {'$set': record}, upsert=True)) result = device_table.bulk_write(requests) # print(result.bulk_api_result) # logger.info(result.bulk_api_result) logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') return result else: # print('Scrape collection error') logger.error('Scrape collection error') def cisco_isakmp_policy(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] output = command_output['output'] # print(output.result) ## log # print('\ncisco_isakmp_policy') logger = logging.getLogger(device_name) logger.info('Run: cisco_isakmp_policy') ## parse and update if not output == 'error': scrape = "" tag_found = False isakmp_policy = [] # split scrape by policies for line in output.result.split('\n'): # print(line) if not tag_found: if 'Global IKE policy' in line: tag_found = True else: scrape += f'{line}\n' # print(scrape) # split policies by suite suite = [] if len(scrape) >0: for line in scrape.split('\n'): if 'Protection suite of priority ' in line: suite.append(f'{line}\n') else: suite[-1] += f'{line}\n' # get suite attributes for s in suite: suite_dict = {} # print(s) for line in s.split('\n'): #print(line) sline = line.lstrip() if 'Protection suite of priority' in sline: priority = sline.split(' ')[4] # print(priority) suite_dict.update({'priority': priority}) if 'encryption algorithm:' in sline: if 'Advanced Encryption Standard' in sline: enc_algo = 'aes' elif 'Three key triple DE' in sline: enc_algo = '3des' elif 'Data Encryption Standard' in sline: enc_algo = 'des' else: enc_algo = 'no_match' if enc_algo != 'no_match': enc_kb = [int(x) for x in sline[sline.find("(")+1:sline.find(")")].split() if x.isdigit()] enc_kb = str(enc_kb[0]) if len(enc_kb) >0 else '' if len(enc_kb) >0: enc_algo = enc_algo + '_' + str(enc_kb) # print(enc_algo) suite_dict.update({'enc_algo': enc_algo}) if 'hash algorithm:' in sline: if 'Secure Hash Standard 2' in sline: hash_algo = 'sha2' elif 'Secure Hash Standard' in sline: hash_algo = 'sha' elif 'Message Digest 5' in sline: hash_algo = 'md5' else: hash_algo = 'no_match' if hash_algo != 'no_match': hash_kb = [int(x) for x in sline[sline.find("(")+1:sline.find(")")].split() if x.isdigit()] hash_kb = str(hash_kb[0]) if len(hash_kb) >0 else '' if len(hash_kb) >0: hash_algo = hash_algo + '_' + str(hash_kb) # print(hash_algo) suite_dict.update({'hash_algo': hash_algo}) if 'authentication method:' in sline: if 'Pre-Shared Key' in sline: auth_type = 'psk' else: auth_type = 'no_match' if 'Diffie-Hellman group:' in sline: dh_group = sline.split('Diffie-Hellman group:')[1].split(' ')[0].lstrip().replace('#', '') dh_group_kb = [int(x) for x in sline[sline.find("(")+1:sline.find(")")].split() if x.isdigit()] dh_group_kb = str(dh_group_kb[0]) if len(dh_group_kb) >0 else '' if len(dh_group_kb) >0: dh_group = dh_group + '_' + str(dh_group_kb) # print(dh_group) suite_dict.update({'dh_group': dh_group}) # print(suite_dict) isakmp_policy.append(suite_dict) # get isakmp policy precedence if len(isakmp_policy) >0: # print(isakmp_policy) doc_update = {'isakmp_policy': isakmp_policy} # print(f'isakmp policy entry count {len(isakmp_policy)}') logger.info(f'isakmp policy entry count {len(isakmp_policy)}') # debug - add highest priority isakmp policy to test match 3des logic # isakmp_policy.append({'priority': '0', 'enc_algo': '3des', 'hash_algo': 'sha', 'dh_group': '14_2048'}) # find highest priority isakmp policy enc_algo highest_priority_policy = sorted([int(x['priority']) for x in isakmp_policy])[0] highest_priority_policy_algo = [x['enc_algo'] for x in isakmp_policy if x['priority'] == str(highest_priority_policy)] if 'des' in highest_priority_policy_algo[0]: # print(highest_priority_policy_algo[0]) doc_update.update({'isakmp_policy_default_p1_3des': True}) else: doc_update.update({'isakmp_policy_default_p1_3des': False}) # print(doc_update['isakmp_policy_default_p1_3des']) query = {'DeviceName': device_name} result = collection.update_one(query, {'$set': doc_update}, upsert=True) # print(dir(result)) logger.info(f'Database: matched_count {result.matched_count} modified_count {result.modified_count}') return result else: # print(f'isakmp policy entry count {len(isakmp_policy)}') logger.info(f'isa policy entry count {len(isakmp_policy)}') else: # print('Scrape collection error') logger.error(f'Scrape collection error') def cisco_transform_set(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_type = device_record['DeviceType'] device_table = collection['temp'][device_name] command = command_output['command'] ## log # print('\ncisco_transform_set') logger = logging.getLogger(device_name) logger.info('Run: cisco_transform_set') def process_transform_set(scrape): # print(scrape) tfs_found = False tfset = [] for line in scrape.split('\n'): sline = line.lstrip() if not tfs_found: if 'Transform sets=' in sline: tfs_found = True pass elif ' } ,' in sline: tfs_name = sline.split(' ')[0].split(':')[0] tfs_encr_algo = sline.replace(' ', ' ').split(' ')[2] tfs_hash_algo = sline.replace(' ', ' ').split(' ')[3] tfset.append({'name': tfs_name, 'p2_encr_algo': tfs_encr_algo, 'p2_hash_algo': tfs_hash_algo}) else: tfs_found = False return tfset ## parse and update if command == 'compound': update_src = [] requests = [] ## dmvpn lookup ordered transform set # print('dmvpn') if device_type in ["IP-VPNHUB", "IP-VCSR-HUB"]: tunnel_interfaces = device_table.distinct('p2_interface') # print(f'Tunnel interfaces {tunnel_interfaces}') logger.info(f'tunnel interfaces {tunnel_interfaces}') if len(tunnel_interfaces) >0: for t in tunnel_interfaces: interface_name = t with Scrapli(**connection) as conn: interface = conn.send_command(f'show interface {interface_name}') parsed = interface.genie_parse_output() # print(json.dumps(parsed, indent=4)) # sometimes find the p2_interface does not match a real interface (thus cannot find ipsec profile), check output is populated # p2_interface were in the temp scrape table then gone on the next scrape - zfr-dmvpn-hub04 - show interface Tunnel45/55 # could this be a broken config and some tunnels that try to establish listing a non existent interface? if len(parsed) >0: if 'tunnel_profile' in parsed[interface_name]: ipsec_profile_name = parsed[interface_name]['tunnel_profile'] elif 'Tunnel protection via IPSec' in interface.result: # some ios genie outputs are not fully parsed, failback to manual parse for line in interface.result.split('\n'): if 'Tunnel protection via IPSec' in line: ipsec_profile_name = [a for a in line[line.find("(")+1:line.find(")")].split()][1] # print(ipsec_profile_name) if 'ipsec_profile_name' in locals(): with Scrapli(**connection) as conn: # command has no genie parser ipsec_profile = conn.send_command(f'show crypto ipsec profile {ipsec_profile_name}') # print(ipsec_profile.result) transform_set = process_transform_set(ipsec_profile.result) match_field = 'p2_interface' match_field_value = t update_src.append({'match_field': match_field, 'match_field_value': match_field_value, 'transform_set': transform_set}) ## rri lookup ordered transform set # print('rri') if device_type in ["IP-VPNAGG", "IP-P2PAGG"]: crypto_map_templates = device_table.distinct('crypto_map_template') # print(f'Crypto map templates {crypto_map_templates}') logger.info(f'crypto map templates {crypto_map_templates}') if len(crypto_map_templates) >0: for t in crypto_map_templates: with Scrapli(**connection) as conn: crypto_map = conn.send_command(f'show crypto dynamic-map tag {t}') # print(crypto_map.result) transform_set = process_transform_set(crypto_map.result) match_field = 'crypto_map_template' match_field_value = t update_src.append({'match_field': match_field, 'match_field_value': match_field_value, 'transform_set': transform_set}) ## build db update requests # print(json.dumps(update_src, indent=4)) if len(update_src) >0: for r in update_src: query = {r['match_field']: r['match_field_value']} # print(query) object_ids = [d for d in device_table.distinct('_id', query)] # print(object_ids) query = { "_id" : { "$in" : object_ids } } update = {'ordered_transform_set': r['transform_set']} requests.append(UpdateMany(query, {'$set': update}, upsert=True)) # print(requests) ## bulk update collection documents with ordered_transform_sets if len(requests) >0: result = device_table.bulk_write(requests) # print(result.bulk_api_result) # logger.info(result.bulk_api_result) logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') return result else: # print("run with 'command': 'compound'") logger.error('Compound command, device_commands function requires command dictionary containing item: "cisco_transform_set": {"command": "compound"}') def triple_des_check(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_table = collection['temp'][device_name] command = command_output['command'] ## log # print('\ntriple_des_check') logger = logging.getLogger(device_name) logger.info('Run: triple_des_check') ## owing to the age of mongodb 3.0.15 some filters/operators are not available, the following queries could otherwise be merged and done in bulk in the query language with a huge performance uptick # "$arrayElemAt" "$first" "$slice", "$regex" also does not honour read ahead negative match (?!3des) # https://stackoverflow.com/questions/29664097/what-is-the-syntax-for-mongodb-query-for-boolean-values # https://www.tutorialspoint.com/get-the-first-element-in-an-array-and-return-using-mongodb-aggregate def p2_encr_algo_check(collection, triple_des_match = True): if triple_des_match: #regex_statement = {'$regex': '.*3des.*', '$options': 'i'} regex_statement = re.compile('(?i).*3DES.*') else: regex_statement = {'$not': re.compile('(?i).*3DES.*')} result = collection.aggregate([ {"$match": {"ordered_transform_set": {"$exists": True}}}, {"$match": {'p2_encr_algo': regex_statement }}, {"$project": {"_id": 1}} ]) matched_doc_ids = [d['_id'] for d in result] # print(dumps(matched_doc_ids, indent=4)) return matched_doc_ids def first_ordered_transform_set_check(collection, doc_ids, triple_des_match = True): matched_doc_ids = [] if triple_des_match: regex_statement = re.compile('(?i).*3DES.*') else: regex_statement = {'$not': re.compile('(?i).*3DES.*')} for doc_id in doc_ids: result = collection.aggregate([ {"$match": { "_id" : doc_id }}, {"$unwind": "$ordered_transform_set"}, {"$limit": 1 }, {"$match": {'ordered_transform_set.p2_encr_algo': regex_statement}}, {"$project": {"_id": 1}} ]) for result_id in [d['_id'] for d in result]: matched_doc_ids.append(result_id) # print(dumps(matched_doc_ids, indent=4)) return matched_doc_ids def tdes_requests_builder(requests, doc_ids, p2_default_3des, spoke_p2_default_3des, spoke_p2_algo_preference): if len(doc_ids) >0: update = {} update.update({'p2_default_3des': p2_default_3des}) if spoke_p2_default_3des != 'unset': update.update({'spoke_p2_default_3des': spoke_p2_default_3des}) update.update({'spoke_p2_algo_preference': spoke_p2_algo_preference}) # print(json.dumps(update, indent=4)) query = { "_id" : { "$in" : doc_ids } } requests.append(UpdateMany(query, {'$set': update}, upsert=True)) return requests ## process and update if command == 'compound': requests = [] #### p2_encr_algo 3des triple_des_match = True tdes_doc_ids = p2_encr_algo_check(device_table, triple_des_match) ## 1st ordered_transform_set 3des triple_des_match = True tdes_tdes_ids = first_ordered_transform_set_check(device_table, tdes_doc_ids, triple_des_match) # p2_default_3des = True / spoke_p2_default_3des = unset / spoke_p2_algo_preference = unknown requests = tdes_requests_builder(requests, tdes_tdes_ids, True, 'unset', 'unknown') ## 1st ordered_transform_set NOT 3des triple_des_match = False tdes_ntdes_ids = first_ordered_transform_set_check(device_table, tdes_doc_ids, triple_des_match) # p2_default_3des False / spoke_p2_default_3des True / spoke_p2_algo_preference = 3des requests = tdes_requests_builder(requests, tdes_ntdes_ids, False, True, '3des') #### p2_encr_algo NOT 3des triple_des_match = False ntdes_doc_ids = p2_encr_algo_check(device_table, triple_des_match) ## 1st ordered_transform_set 3des triple_des_match = True ntdes_tdes_ids = first_ordered_transform_set_check(device_table, ntdes_doc_ids, triple_des_match) # p2_default_3des True / spoke_p2_default_3des False / spoke_p2_algo_preference = not 3des requests = tdes_requests_builder(requests, ntdes_tdes_ids, True, False, 'not 3des') ## 1st ordered_transform_set NOT 3des triple_des_match = False ntdes_ntdes_ids = first_ordered_transform_set_check(device_table, ntdes_doc_ids, triple_des_match) # p2_default_3des False / spoke_p2_default_3des unset / spoke_p2_algo_preference = unknown requests = tdes_requests_builder(requests, ntdes_ntdes_ids, False, 'unset', 'unknown') ## bulk update collection documents with ordered_transform_sets if len(requests) >0: result = device_table.bulk_write(requests) # print(result.bulk_api_result) # logger.info(result.bulk_api_result) logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') else: # print("run with 'command': 'compound'") logger.error('Compound command, device_commands function requires command dictionary containing item: "triple_des_check": {"command": "compound"}') def cisco_nhrp_lookup(collection, command_output, device_record, connection): ## init device_name = device_record['DeviceName'] device_type = device_record['DeviceType'] device_table = collection['temp'][device_name] command = command_output['command'] ## log logger = logging.getLogger(device_name) ## parse and update if command == 'compound': # vars here nhrp_map = [] unique_nhrp_map = [] requests = [] ## dmvpn lookup nhrp address mapping if device_type in ["IP-VPNHUB", "IP-VCSR-HUB"]: logger.info('Run: cisco_nhrp_lookup') with Scrapli(**connection) as conn: nhrp = conn.send_command(f'show ip nhrp brief') # logger.info(f'{nhrp.result}') ## process nhrp address mappings scrape for line in nhrp.result.split('\n'): line_fields = ' '.join(line.split()).split(' ') if len(line_fields) == 3: if valid_ipv4(line_fields[1]) and valid_ipv4(line_fields[2]): # logger.info(f"{line_fields[1]} {line_fields[2]}") nbma_address = line_fields[2] nhrp_nexthop = line_fields[1] nhrp_map.append({'nbma_address': nbma_address, 'nhrp_nexthop': nhrp_nexthop}) ## ensure no duplicates (the nature of the 1:1 address mapping with nhrp likely negates the need for this check) for entry in nhrp_map: if entry not in unique_nhrp_map: unique_nhrp_map.append(entry) ## write to db if len(unique_nhrp_map) >0: for record in unique_nhrp_map: #filter = {'peer_vpn_id': record['nbma_address']} # 'peer_vpn_id' is just a handshake attribute set by the engineer, 'peer_ip' is a an immutable fact in the nhrp map filter = {'peer_ip': record['nbma_address']} update = {'peer_vpn_id': record['nbma_address'], 'nhrp_nexthop': record['nhrp_nexthop']} requests.append(UpdateMany(filter, {'$set': update}, upsert=False)) # upsert should be false, only looking to update records that match, not insert records with only 'peer_vpn_id' + 'nhrp_nexthop' fields result = device_table.bulk_write(requests) logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}') return result else: # print("run with 'command': 'compound'") logger.error('Compound command, device_commands function requires command dictionary containing item: "cisco_peer_lookup": {"command": "compound"}')