894 lines
39 KiB
Python
894 lines
39 KiB
Python
import os
|
|
from functools import partial
|
|
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
|
|
from scrapli import Scrapli
|
|
# from scrapli.driver import GenericDriver
|
|
# from scrapli.driver.core import IOSXEDriver
|
|
from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne, UpdateMany
|
|
import re
|
|
import json
|
|
|
|
def cisco_version(collection, command_output, device_record, connection):
|
|
print('run cisco_version')
|
|
def device_audit():
|
|
return {'os_flavour': operating_system, 'image': image, 'os_version': version, 'chassis': chassis, 'serial': serial}
|
|
id = device_record['_id']
|
|
output = command_output['output']
|
|
if not output == 'error':
|
|
if len(output.genie_parse_output()) >0:
|
|
print(f'genie parser method')
|
|
parsed = output.genie_parse_output()
|
|
operating_system = parsed['version']['os']
|
|
image = parsed['version']['system_image'].split(':')[1].replace('/', '')
|
|
version = parsed['version']['version']
|
|
chassis = parsed['version']['chassis']
|
|
serial = parsed['version']['chassis_sn']
|
|
record = device_audit()
|
|
# print(record)
|
|
filter = {'_id': id}
|
|
result = collection.update_one(filter, {'$set': record}, upsert=True)
|
|
return result
|
|
else:
|
|
print(f'manual parse method not written for this command')
|
|
|
|
#### scrape 1)
|
|
# 'show isakmp sa detail'
|
|
# {'c_id': '20681', 'local_ip': '10.227.184.157', 'p1_ivrf': 'none', 'peer_ip': '10.229.4.74', 'p1_dh_group': '14', 'p1_encr_algo': 'aes', 'p1_hash_algo': 'sha', 'p1_auth_type': 'psk', 'p1_status': 'ACTIVE'}
|
|
def cisco_vpn_phase1(collection, command_output, device_record, connection):
|
|
print('\ncisco_vpn_phase1')
|
|
|
|
def process_p1(p1_dict, idx):
|
|
global peer_count
|
|
global p1_records
|
|
c_id = str(p1_dict['isakmp_stats']['IPv4'][idx]['c_id'])
|
|
local_ip = p1_dict['isakmp_stats']['IPv4'][idx]['local_ip']
|
|
peer_ip = p1_dict['isakmp_stats']['IPv4'][idx]['remote_ip']
|
|
encr_algo = p1_dict['isakmp_stats']['IPv4'][idx]['encr_algo']
|
|
hash_algo = p1_dict['isakmp_stats']['IPv4'][idx]['hash_algo']
|
|
auth_type = p1_dict['isakmp_stats']['IPv4'][idx]['auth_type']
|
|
dh_group = str(p1_dict['isakmp_stats']['IPv4'][idx]['dh_group'])
|
|
status = p1_dict['isakmp_stats']['IPv4'][idx]['status']
|
|
ivrf = p1_dict['isakmp_stats']['IPv4'][idx]['ivrf'] if 'ivrf' in p1_dict['isakmp_stats']['IPv4'][idx] else 'none'
|
|
p1_record = {'c_id': c_id, 'local_ip': local_ip, 'p1_ivrf': ivrf, 'peer_ip': peer_ip, 'p1_dh_group': dh_group, 'p1_encr_algo': encr_algo, 'p1_hash_algo': hash_algo, 'p1_auth_type': auth_type, 'p1_status': status }
|
|
# print(f'genie p1 {p1_records}')
|
|
p1_records.append(p1_record)
|
|
peer_count += 1
|
|
print('phase1 processed %d\r'%peer_count, end="")
|
|
|
|
device_name = device_record['DeviceName']
|
|
output = command_output['output']
|
|
device_table = collection['temp'][device_name] # create/use-existing temp subcollection
|
|
if not output == 'error':
|
|
p1_dict = output.genie_parse_output()
|
|
# print(json.dumps(p1_dict, indent=4))
|
|
if len(p1_dict) >0:
|
|
global peer_count
|
|
peer_count = 0
|
|
global p1_records
|
|
p1_records = []
|
|
p1_scrape_idx = [t for t in p1_dict['isakmp_stats']['IPv4']]
|
|
n_cores = os.cpu_count()
|
|
partial_function = partial(process_p1, p1_dict)
|
|
# with ThreadPoolExecutor(max_workers=1) as executor: # debug
|
|
with ThreadPoolExecutor(max_workers=n_cores) as executor:
|
|
executor.map(partial_function, p1_scrape_idx)
|
|
print(f'phase1 processed {peer_count}')
|
|
# write to db
|
|
if len(p1_records) >0:
|
|
requests = []
|
|
for i in p1_records:
|
|
record = i
|
|
# requests.append(InsertOne(record))
|
|
filter = {'c_id': record['c_id']}
|
|
requests.append(UpdateMany(filter, {'$set': record}, upsert=True))
|
|
result = device_table.bulk_write(requests)
|
|
# print(result.bulk_api_result)
|
|
return result
|
|
else:
|
|
print('phase1 no tunnel records')
|
|
else:
|
|
print('error returning command, check network connectivity')
|
|
|
|
#### scrape 2)
|
|
# 'show crypto session remote {ip} detail'
|
|
# {'local_ip': '10.225.112.42', 'local_port': '500', 'c_id': '11907', 'ipsec_flow': ['permit 47 host 10.225.112.42 host 10.227.36.18'], 'crypto_session_interface': 'Tunnel6', 'session_status': 'UP-ACTIVE', 'peer_ip': '10.227.36.18', 'peer_port': '500', 'p2_fvrf': 'none', 'peer_vpn_id': '10.227.36.18'}
|
|
# correlate to scrape 1) with key 'c_id'
|
|
def cisco_crypto_session(collection, command_output, device_record, connection):
|
|
print('\ncisco_crypto_session')
|
|
|
|
def process_session(session):
|
|
# debug with single thread
|
|
# print('\n##########')
|
|
# print(session)
|
|
if not 'Invalid input detected at' in session: # occurs with no match on peer ip
|
|
# wipe all lines before first match of 'Interface: ', this match is used to delimit lines of text into subsequent interface scrapes
|
|
scrape = ""
|
|
tag_found = False
|
|
for line in session.split('\n'):
|
|
if not tag_found:
|
|
if 'Interface: ' in line:
|
|
scrape += f'{line}\n'
|
|
tag_found = True
|
|
else:
|
|
scrape += f'{line}\n'
|
|
|
|
# split scrapes into multiple interfaces, each interface entry may have many sessions
|
|
interfaces = []
|
|
try:
|
|
if len(scrape) >0:
|
|
for line in scrape.split('\n'): # this can fail but only on a huge scrape and hard to see, mep-shared-rri-agg09
|
|
if 'Interface: ' in line:
|
|
interfaces.append(f'{line}\n')
|
|
else:
|
|
interfaces[-1] += f'{line}\n'
|
|
except Exception as e:
|
|
print(f'failed to process scrape: {e}')
|
|
#print(scrape)
|
|
pass
|
|
# print(f'retrieved crypto session interface entries {len(interfaces)}')
|
|
|
|
# loop interfaces, loop session attributes
|
|
global task_count
|
|
global session_records
|
|
for i in interfaces:
|
|
#print(i)
|
|
peer_record_dict = {}
|
|
session_record_dict = {}
|
|
#global task_count
|
|
all_sess = ""
|
|
all_sess_found = False
|
|
for line in i.split('\n'):
|
|
if 'Interface: ' in line:
|
|
crypto_session_interface = line.split(' ')[1]
|
|
peer_record_dict.update({'crypto_session_interface': crypto_session_interface})
|
|
if 'Profile: ' in line:
|
|
p1_profile = line.split(' ')[1]
|
|
peer_record_dict.update({'p1_profile': p1_profile})
|
|
if 'Session status: ' in line:
|
|
session_status = line.split('Session status: ')[1]
|
|
peer_record_dict.update({'session_status': session_status})
|
|
if 'Peer: ' in line:
|
|
peer_ip = line.split(' ')[1]
|
|
peer_port = line.split(' ')[3]
|
|
p2_fvrf = line.split(' ')[5].replace('(', '').replace(')', '')
|
|
# p1_vrf = line.split(' ')[7]
|
|
peer_record_dict.update({'peer_ip': peer_ip, 'peer_port': peer_port, 'p2_fvrf': p2_fvrf})
|
|
if 'Phase1_id: ' in line:
|
|
peer_vpn_id = line.lstrip().split(' ')[1].replace('(', '').replace(')', '')
|
|
peer_record_dict.update({'peer_vpn_id': peer_vpn_id})
|
|
# split all lines from 'IKEv1 SA: ' to end
|
|
if not all_sess_found:
|
|
if any(ike in line for ike in ['IKEv1 SA: ', 'IKE SA: ']):
|
|
all_sess += f'{line}\n'
|
|
all_sess_found = True
|
|
elif 'Session ID: ' in line:
|
|
pass
|
|
else:
|
|
all_sess += f'{line}\n'
|
|
|
|
# breakout each session, this can be P1 only with 'IKEv1 SA: ' or P1 + P2 with 'IKEv1 SA: ' and 'IPSEC FLOW: '
|
|
all_sess_pairs = []
|
|
for line in all_sess.split('\n'):
|
|
if any(ike in line for ike in ['IKEv1 SA: ', 'IKE SA: ']):
|
|
all_sess_pairs.append(f'{line}\n')
|
|
else:
|
|
all_sess_pairs[-1] += f'{line}\n'
|
|
for asp in all_sess_pairs:
|
|
#print(asp)
|
|
ipsec_flow =[]
|
|
for line in asp.split('\n'):
|
|
# if 'IKEv1 SA: ' in line: # fails for older cisco
|
|
if any(ike in line for ike in ['IKEv1 SA: ', 'IKE SA: ']):
|
|
local_ip = line.lstrip().split(' ')[3].split('/')[0]
|
|
local_port = line.lstrip().split(' ')[3].split('/')[1]
|
|
session_record_dict.update({'local_ip': local_ip, 'local_port': local_port})
|
|
if 'connid:' in line:
|
|
c_id = line.lstrip().split(' ')[1].split(':')[1]
|
|
session_record_dict.update({'c_id': c_id})
|
|
if 'IPSEC FLOW: ' in line:
|
|
acl = line.lstrip().split('FLOW: ')[1]
|
|
ipsec_flow.append(acl)
|
|
if len(ipsec_flow) >0:
|
|
session_record_dict.update({'ipsec_flow': ipsec_flow})
|
|
|
|
# merge peer record component with each session record
|
|
session_record_dict.update(peer_record_dict)
|
|
# print(session_record_dict)
|
|
|
|
# check for required fields for complete/valid session record
|
|
if all(m in session_record_dict for m in ['local_ip', 'peer_ip', 'c_id']):
|
|
task_count += 1
|
|
print('session interface entries processed %d\r'%task_count, end="")
|
|
session_records.append(session_record_dict)
|
|
|
|
# init vars, start
|
|
device_name = device_record['DeviceName']
|
|
device_table = collection['temp'][device_name] # create/use-existing temp subcollection
|
|
peer_ips = []
|
|
sessions = []
|
|
session_count = 0
|
|
global session_records
|
|
session_records = []
|
|
global task_count
|
|
task_count = 0
|
|
|
|
# get scrapes
|
|
with Scrapli(**connection) as conn:
|
|
peers = conn.send_command('show crypto session brief')
|
|
for line in peers.result.split('\n'):
|
|
if len(line) >0 and not any(exclude in line for exclude in ['ivrf = ', 'Peer ', 'Status: ', 'No IKE']):
|
|
# print(line)
|
|
format_line = ' '.join(line.split())
|
|
ip = format_line.split(' ')[0]
|
|
# print(ip)
|
|
peer_ips.append(format_line.split(' ')[0])
|
|
if len(peer_ips) >0:
|
|
print(f'crypto session count {len(peer_ips)}')
|
|
for ip in peer_ips:
|
|
session_count += 1
|
|
print('lookup crypto sessions %d\r'%session_count, end="")
|
|
session = conn.send_command(f'show crypto session remote {ip} detail')
|
|
# print(session.result)
|
|
sessions.append(session.result)
|
|
|
|
# process scrapes
|
|
print(f'lookup crypto sessions {len(sessions)}')
|
|
if len(sessions) >0:
|
|
n_cores = os.cpu_count()
|
|
partial_function = partial(process_session)
|
|
#with ThreadPoolExecutor(max_workers=1) as executor: # debug
|
|
with ThreadPoolExecutor(max_workers=n_cores) as executor:
|
|
executor.map(partial_function, sessions)
|
|
print(f'session interface entries processed {task_count}')
|
|
|
|
# write to db
|
|
if len(session_records) >0:
|
|
requests = []
|
|
for i in session_records:
|
|
record = i
|
|
filter = {'c_id': record['c_id']}
|
|
requests.append(UpdateMany(filter, {'$set': record}, upsert=True))
|
|
result = device_table.bulk_write(requests)
|
|
print(result.bulk_api_result)
|
|
return result
|
|
|
|
#### scrape 3)
|
|
# 'show crypto ipsec sa peer {p}'
|
|
# {'p2_interface': 'Tunnel6', 'local_ip': '10.225.112.42', 'peer_ip': '10.225.56.110', 'peer_port': '500', 'protected_vrf': 'none', 'pfs': 'N', 'p2_encr_algo': 'esp-256-aes', 'p2_hash_algo': 'esp-sha-hmac', 'p2_status': 'ACTIVE', 'crypto_map': 'Tunnel6-head-0'}
|
|
# correlate to (scrape 1) + scrape 2)) with keys 'local_ip' 'peer_ip' 'peer_port'
|
|
def cisco_vpn_phase2(collection, command_output, device_record, connection):
|
|
print('\ncisco_vpn_phase2')
|
|
|
|
def process_p2_scrapes(record):
|
|
global p2_scrapes
|
|
global empty_p2_scrapes
|
|
global p2_records
|
|
interfaces = []
|
|
if not len(record) >0:
|
|
empty_p2_scrapes += 1
|
|
else:
|
|
for line in record.split('\n'):
|
|
if 'interface: ' in line:
|
|
interfaces.append(f'{line}\n')
|
|
else:
|
|
interfaces[-1] += f'{line}\n'
|
|
# print(f'manual scrape tunnel interface count {len(interfaces)}')
|
|
for int in interfaces:
|
|
# reset interface vars
|
|
for va in ['p2_interface', 'local_ip']:
|
|
if va in locals():
|
|
del va
|
|
# get interface vars
|
|
for line in int.split('\n'):
|
|
if 'interface' in line:
|
|
p2_interface = line.split(' ')[1]
|
|
if 'local addr' in line:
|
|
local_ip = line.split('addr ')[1]
|
|
# strip up to 'protected vrf:' for ivrf loop
|
|
intf = ""
|
|
tag_found = False
|
|
for line in int.split('\n'):
|
|
if not tag_found:
|
|
if 'protected vrf:' in line:
|
|
intf += f'{line}\n'
|
|
tag_found = True
|
|
else:
|
|
intf += f'{line}\n'
|
|
# loop vrfs
|
|
vrfs = []
|
|
for line in intf.split('\n'):
|
|
if 'protected vrf:' in line:
|
|
vrfs.append(f'{line}\n')
|
|
else:
|
|
vrfs[-1] += f'{line}\n'
|
|
for v in vrfs:
|
|
# reset ivrf vars
|
|
for va in ['peer_ip', 'peer_port', 'vrf', 'pfs', 'transform', 'p2_encr_algo', 'p2_hash_algo', 'status', 'crypto_map']:
|
|
if va in locals():
|
|
del va
|
|
peer_ip_l = []
|
|
peer_port_l = []
|
|
vrf_l = []
|
|
pfs_l = []
|
|
transform_l = []
|
|
status_l = []
|
|
crypto_map_l = []
|
|
p2_record_dict = {}
|
|
|
|
# get vrf vars
|
|
for line in v.split('\n'):
|
|
if 'current_peer' in line:
|
|
peer_ip_l.append(line.lstrip(' ').split(' ')[1])
|
|
peer_port_l.append(line.lstrip(' ').split(' ')[3])
|
|
if 'PFS' in line:
|
|
pfs_l.append(line.lstrip(' ').split(' ')[2].split(',')[0].lower())
|
|
if 'transform' in line:
|
|
transform_l.append(line.lstrip(' ').split('transform: ')[1].split(' ,')[0])
|
|
if 'crypto map: ' in line:
|
|
crypto_map_l.append(line.lstrip(' ').split('crypto map: ')[1])
|
|
if 'Status' in line:
|
|
status_l.append(line.split('Status: ')[1].split('(')[0])
|
|
if 'protected vrf' in line:
|
|
vrf_l.append(line.split('protected vrf: ')[1].replace('(', '').replace(')', ''))
|
|
|
|
# write vrf vars to record dict
|
|
p2_record_dict.update({'p2_interface': p2_interface})
|
|
p2_record_dict.update({'local_ip': local_ip})
|
|
if len(peer_ip_l) >0:
|
|
#peer_ip = peer_ip_l[0]
|
|
p2_record_dict.update({'peer_ip': peer_ip_l[0]})
|
|
if len(peer_port_l) >0:
|
|
#peer_port = peer_port_l[0]
|
|
p2_record_dict.update({'peer_port': peer_port_l[0]})
|
|
if len(vrf_l) >0:
|
|
#vrf = vrf_l[0]
|
|
p2_record_dict.update({'protected_vrf': vrf_l[0]})
|
|
if len(pfs_l) >0:
|
|
#pfs = pfs_l[0].upper()
|
|
p2_record_dict.update({'pfs': pfs_l[0].upper()})
|
|
# else:
|
|
# pfs = 'N'
|
|
if len(transform_l) >0:
|
|
transform = transform_l[0]
|
|
#p2_encr_algo = transform.split(' ')[0]
|
|
p2_record_dict.update({'p2_encr_algo': transform.split(' ')[0]})
|
|
#p2_hash_algo = transform.split(' ')[1]
|
|
p2_record_dict.update({'p2_hash_algo': transform.split(' ')[1]})
|
|
if len(status_l) >0:
|
|
#status = status_l[0]
|
|
p2_record_dict.update({'p2_status': status_l[0]})
|
|
if len(crypto_map_l) >0:
|
|
#crypto_map = crypto_map_l[0]
|
|
p2_record_dict.update({'crypto_map': crypto_map_l[0]})
|
|
# print(p2_record_dict)
|
|
|
|
# check for required fields for complete/valid p2 record
|
|
if all(include in p2_record_dict for include in ['local_ip', 'peer_ip', 'peer_port']):
|
|
# print(p2_record_dict)
|
|
p2_records.append(p2_record_dict)
|
|
p2_scrapes += 1
|
|
print('phase2 scrape processed %d\r'%p2_scrapes, end="")
|
|
|
|
def get_p2_scrapes(peers, connection):
|
|
global scrape_count
|
|
peer_commands = [f'show crypto ipsec sa peer {p}' for p in set(peers)]
|
|
print(f'phase2 peer commands prepared {len(peer_commands)}')
|
|
p2scrapes = []
|
|
if len(peers) >0:
|
|
try:
|
|
with Scrapli(**connection) as conn:
|
|
for c in peer_commands:
|
|
p2scrape = conn.send_command(c)
|
|
p2scrapes.append(p2scrape)
|
|
scrape_count += 1
|
|
print('lookup phase2 peers %d\r'%scrape_count, end="")
|
|
return p2scrapes
|
|
except Exception as e:
|
|
print(f'exception_type: {type(e).__name__}')
|
|
# pass
|
|
print('phase2 failback parser method error collecting scrapes, possible incomplete records for this device')
|
|
return p2scrapes
|
|
else:
|
|
print('phase2 failback parser method error, no phase1 peers/tunnels')
|
|
return p2scrapes
|
|
|
|
# init vars
|
|
device_name = device_record['DeviceName']
|
|
device_table = collection['temp'][device_name] # create/use-existing temp subcollection
|
|
global scrape_count
|
|
scrape_count = 0
|
|
global p2_scrapes
|
|
p2_scrapes = 0
|
|
global empty_p2_scrapes
|
|
empty_p2_scrapes = 0
|
|
global p2_records
|
|
p2_records = []
|
|
|
|
# lookup all peer_ip
|
|
peer_ip_query = device_table.find({"peer_ip": { "$exists": True }}, {"peer_ip":1, "_id":0})
|
|
peers = [r['peer_ip'] for r in peer_ip_query]
|
|
|
|
# get p2 scrapes
|
|
result = get_p2_scrapes(peers, connection)
|
|
print(f'lookup phase2 peers {scrape_count}')
|
|
|
|
# process p2 scrapes
|
|
if len(result) >0:
|
|
scrape_results = [s.result for s in result]
|
|
n_cores = os.cpu_count()
|
|
partial_function = partial(process_p2_scrapes)
|
|
# with ThreadPoolExecutor(max_workers=1) as executor: # debug
|
|
with ThreadPoolExecutor(max_workers=n_cores) as executor:
|
|
executor.map(partial_function, scrape_results)
|
|
print(f'phase2 scrape processed {p2_scrapes}')
|
|
print(f'empty phase2 scrapes {empty_p2_scrapes}')
|
|
|
|
# write to db
|
|
if len(p2_records) >0:
|
|
requests = []
|
|
for i in p2_records:
|
|
record = i
|
|
filter = {'local_ip': record['local_ip'], 'peer_ip': record['peer_ip'], 'peer_port': record['peer_port']}
|
|
requests.append(UpdateMany(filter, {'$set': record}, upsert=True))
|
|
result = device_table.bulk_write(requests)
|
|
print(result.bulk_api_result)
|
|
return result
|
|
|
|
#### scrape 4)
|
|
# {'crypto_map': 'Tunnel6-head-0', 'peer_ip': '10.227.112.50', 'pfs': 'N', 'transform_sets': [{'name': 'TS-AES256-SHA', 'p2_encr_algo': 'esp-256-aes', 'p2_hash_algo': 'esp-sha-hmac'}, {'name': 'TS-3DES-SHA', 'p2_encr_algo': 'esp-3des', 'p2_hash_algo': 'esp-sha-hmac'}], 'crypto_map_interface': ['Tunnel6'], 'RRI_enabled': False, 'default_p2_3des': False}
|
|
# correlate to (scrape 1) + scrape 2) + scrape 3)) with keys 'peer_ip' 'crypto_map'
|
|
def cisco_crypto_map(collection, command_output, device_record, connection):
|
|
print('\ncisco_crypto_map')
|
|
|
|
def process_cryptomaps(cryptomap):
|
|
# print(cryptomap)
|
|
# loop lines in cryptomap entry
|
|
global crypto_map_count
|
|
global cryptomap_records
|
|
tfs_found = False
|
|
int_found = False
|
|
tfset = []
|
|
crypto_map_interface = []
|
|
cryptomap_record_dict = {}
|
|
for line in cryptomap.split('\n'):
|
|
if 'Crypto Map "' in line: # older variant of ios
|
|
crypto_map = line.split(' ')[2].replace('"', '')
|
|
cryptomap_record_dict.update({'crypto_map': crypto_map})
|
|
if 'Crypto Map IPv4 "' in line: # newer variant of ios
|
|
crypto_map = line.split(' ')[3].replace('"', '')
|
|
cryptomap_record_dict.update({'crypto_map': crypto_map})
|
|
if 'ISAKMP Profile: ' in line:
|
|
p1_profile = line.split(' ')[2]
|
|
cryptomap_record_dict.update({'p1_profile': p1_profile})
|
|
if 'Current peer: ' in line:
|
|
peer_ip = line.split(' ')[2]
|
|
cryptomap_record_dict.update({'peer_ip': peer_ip})
|
|
# RRI devices use dynamic crypto map templates, the name of the crypto map may not match the template name CM-BML-RRI != CDM-BML-RRI
|
|
if 'dynamic (created from dynamic map ' in line:
|
|
# dynamic (created from dynamic map CDM-BML-RRI/200)
|
|
crypto_map_template = line.split('dynamic map ')[1].split('/')[0]
|
|
cryptomap_record_dict.update({'crypto_map_template': crypto_map_template})
|
|
if 'PFS (Y/N): ' in line:
|
|
pfs = line.split(' ')[2].upper()
|
|
cryptomap_record_dict.update({'pfs': pfs})
|
|
|
|
if not tfs_found:
|
|
if 'Transform sets=' in line:
|
|
tfs_found = True
|
|
pass
|
|
elif ' } ,' in line:
|
|
tfs_name = line.split(' ')[0].split(':')[0]
|
|
tfs_encr_algo = line.replace(' ', ' ').split(' ')[2]
|
|
tfs_hash_algo = line.replace(' ', ' ').split(' ')[3]
|
|
tfset.append({'name': tfs_name, 'p2_encr_algo': tfs_encr_algo, 'p2_hash_algo': tfs_hash_algo})
|
|
else:
|
|
tfs_found = False
|
|
|
|
if 'Reverse Route Injection Enabled' in line:
|
|
cryptomap_record_dict.update({'RRI_enabled': True})
|
|
|
|
if not int_found:
|
|
if 'Interfaces using crypto map ' in line:
|
|
int_found = True
|
|
pass
|
|
else:
|
|
if len(line) >0:
|
|
crypto_map_interface.append(line)
|
|
|
|
# add possible list items to cryptomap record
|
|
if len(tfset) >0:
|
|
cryptomap_record_dict.update({'transform_sets': tfset})
|
|
if len(crypto_map_interface) >0:
|
|
cryptomap_record_dict.update({'crypto_map_interface' : crypto_map_interface})
|
|
|
|
# catch absence of RRI
|
|
if 'RRI_enabled' not in cryptomap_record_dict:
|
|
cryptomap_record_dict.update({'RRI_enabled': False})
|
|
|
|
# # DISABLE - transform_sets is dynamic, not the best source of truth
|
|
# # determine if 1st/default P2 transform set is 3des
|
|
# if 'transform_sets' in cryptomap_record_dict:
|
|
# if '3des' in cryptomap_record_dict['transform_sets'][0]['p2_encr_algo'].lower():
|
|
# cryptomap_record_dict.update({'default_p2_3des': True})
|
|
# else:
|
|
# cryptomap_record_dict.update({'default_p2_3des': False})
|
|
# # print(cryptomap_record_dict)
|
|
|
|
# check for required fields for complete/valid cryptomap record (if the cryptomap has no peer_ip it has no use)
|
|
if all(include in cryptomap_record_dict for include in ['peer_ip', 'crypto_map']):
|
|
# print(cryptomap_record_dict)
|
|
cryptomap_records.append(cryptomap_record_dict)
|
|
crypto_map_count += 1
|
|
print('cryptomaps processed %d\r'%crypto_map_count, end="")
|
|
|
|
# start, init vars
|
|
global crypto_map_count
|
|
crypto_map_count = 0
|
|
global cryptomap_records
|
|
cryptomap_records = []
|
|
device_name = device_record['DeviceName']
|
|
device_table = collection['temp'][device_name] # create/use-existing temp subcollection
|
|
output = command_output['output']
|
|
# print(output.result)
|
|
|
|
if output == 'error':
|
|
print('parse failure, output too large for screen scrape, consider a command targetted by peer')
|
|
elif output == 'compound':
|
|
print('cisco_crypto_map is not a compound command')
|
|
else:
|
|
# strip everything up to first 'Crypto Map IPv4 '
|
|
scrape = ""
|
|
tag_found = False
|
|
for line in output.result.split('\n'):
|
|
if not tag_found:
|
|
#if 'Crypto Map IPv4 ' in line:
|
|
if 'Crypto Map ' in line:
|
|
scrape += f'{line}\n'
|
|
tag_found = True
|
|
else:
|
|
scrape += f'{line}\n'
|
|
# print(scrape)
|
|
|
|
# split document into cryptomap entries
|
|
cryptomaps = []
|
|
crypto_map_found_count = 0
|
|
try:
|
|
if len(scrape) >0:
|
|
for line in scrape.split('\n'): # this can fail but only on a huge scrape which is hard to see - mep-shared-rri-agg09
|
|
# if 'Crypto Map IPv4 ' in line: # will not work on older cisco
|
|
if 'Crypto Map ' in line:
|
|
cryptomaps.append(f'{line}\n')
|
|
crypto_map_found_count += 1
|
|
print('lookup crypto maps %d\r'%crypto_map_found_count, end="")
|
|
elif 'Crypto Map: ' in line: # these lines list the isakmp profile for the ipsec cryptomap profile, shorthand output that is not required
|
|
pass
|
|
else:
|
|
cryptomaps[-1] += f'{line.lstrip()}\n'
|
|
print(f'lookup crypto maps {crypto_map_found_count}')
|
|
except Exception as e:
|
|
print(f'failed to process cryptomap scrape: {e}')
|
|
# print(scrape)
|
|
pass
|
|
|
|
# process cryptomap scrapes
|
|
n_cores = os.cpu_count()
|
|
partial_function = partial(process_cryptomaps)
|
|
# with ThreadPoolExecutor(max_workers=1) as executor: # debug
|
|
with ThreadPoolExecutor(max_workers=n_cores) as executor:
|
|
executor.map(partial_function, cryptomaps)
|
|
print(f'cryptomaps processed {crypto_map_count}')
|
|
|
|
# write to db
|
|
if len(cryptomap_records) >0:
|
|
requests = []
|
|
for i in cryptomap_records:
|
|
record = i
|
|
filter = {'peer_ip': record['peer_ip'], 'crypto_map': record['crypto_map']}
|
|
requests.append(UpdateMany(filter, {'$set': record}, upsert=True))
|
|
result = device_table.bulk_write(requests)
|
|
print(result.bulk_api_result)
|
|
return result
|
|
|
|
def cisco_isakmp_policy(collection, command_output, device_record, connection):
|
|
print('\ncisco_isakmp_policy')
|
|
device_name = device_record['DeviceName']
|
|
ip = device_record['IPAddress']
|
|
output = command_output['output']
|
|
|
|
if not output == 'error':
|
|
scrape = ""
|
|
tag_found = False
|
|
isakmp_policy = []
|
|
# split scrape by policies
|
|
for line in output.result.split('\n'):
|
|
# print(line)
|
|
if not tag_found:
|
|
if 'Global IKE policy' in line:
|
|
tag_found = True
|
|
else:
|
|
scrape += f'{line}\n'
|
|
# print(scrape)
|
|
|
|
# split policies by suite
|
|
suite = []
|
|
if len(scrape) >0:
|
|
for line in scrape.split('\n'):
|
|
if 'Protection suite of priority ' in line:
|
|
suite.append(f'{line}\n')
|
|
else:
|
|
suite[-1] += f'{line}\n'
|
|
|
|
# get suite attributes
|
|
for s in suite:
|
|
suite_dict = {}
|
|
# print(s)
|
|
for line in s.split('\n'):
|
|
#print(line)
|
|
sline = line.lstrip()
|
|
if 'Protection suite of priority' in sline:
|
|
priority = sline.split(' ')[4]
|
|
# print(priority)
|
|
suite_dict.update({'priority': priority})
|
|
if 'encryption algorithm:' in sline:
|
|
if 'Advanced Encryption Standard' in sline:
|
|
enc_algo = 'aes'
|
|
elif 'Three key triple DE' in sline:
|
|
enc_algo = '3des'
|
|
elif 'Data Encryption Standard' in sline:
|
|
enc_algo = 'des'
|
|
else:
|
|
enc_algo = 'no_match'
|
|
if enc_algo != 'no_match':
|
|
enc_kb = [int(x) for x in sline[sline.find("(")+1:sline.find(")")].split() if x.isdigit()]
|
|
enc_kb = str(enc_kb[0]) if len(enc_kb) >0 else ''
|
|
if len(enc_kb) >0:
|
|
enc_algo = enc_algo + '_' + str(enc_kb)
|
|
# print(enc_algo)
|
|
suite_dict.update({'enc_algo': enc_algo})
|
|
if 'hash algorithm:' in sline:
|
|
if 'Secure Hash Standard 2' in sline:
|
|
hash_algo = 'sha2'
|
|
elif 'Secure Hash Standard' in sline:
|
|
hash_algo = 'sha'
|
|
elif 'Message Digest 5' in sline:
|
|
hash_algo = 'md5'
|
|
else:
|
|
hash_algo = 'no_match'
|
|
if hash_algo != 'no_match':
|
|
hash_kb = [int(x) for x in sline[sline.find("(")+1:sline.find(")")].split() if x.isdigit()]
|
|
hash_kb = str(hash_kb[0]) if len(hash_kb) >0 else ''
|
|
if len(hash_kb) >0:
|
|
hash_algo = hash_algo + '_' + str(hash_kb)
|
|
# print(hash_algo)
|
|
suite_dict.update({'hash_algo': hash_algo})
|
|
if 'authentication method:' in sline:
|
|
if 'Pre-Shared Key' in sline:
|
|
auth_type = 'psk'
|
|
else:
|
|
auth_type = 'no_match'
|
|
if 'Diffie-Hellman group:' in sline:
|
|
dh_group = sline.split('Diffie-Hellman group:')[1].split(' ')[0].lstrip().replace('#', '')
|
|
dh_group_kb = [int(x) for x in sline[sline.find("(")+1:sline.find(")")].split() if x.isdigit()]
|
|
dh_group_kb = str(dh_group_kb[0]) if len(dh_group_kb) >0 else ''
|
|
if len(dh_group_kb) >0:
|
|
dh_group = dh_group + '_' + str(dh_group_kb)
|
|
# print(dh_group)
|
|
suite_dict.update({'dh_group': dh_group})
|
|
# print(suite_dict)
|
|
isakmp_policy.append(suite_dict)
|
|
|
|
# get isakmp policy precedence
|
|
if len(isakmp_policy) >0:
|
|
# print(isakmp_policy)
|
|
print(f'isakmp policy entry count {len(isakmp_policy)}')
|
|
update = {'isakmp_policy': isakmp_policy}
|
|
# isakmp_policy.append({'priority': '0', 'enc_algo': '3des', 'hash_algo': 'sha', 'dh_group': '14_2048'}) # debug
|
|
# find highest priority isakmp policy enc_algo
|
|
highest_priority_policy = sorted([int(x['priority']) for x in isakmp_policy])[0]
|
|
highest_priority_policy_algo = [x['enc_algo'] for x in isakmp_policy if x['priority'] == str(highest_priority_policy)]
|
|
if 'des' in highest_priority_policy_algo[0]:
|
|
# print(highest_priority_policy_algo[0])
|
|
update.update({'isakmp_policy_default_p1_3des': True})
|
|
else:
|
|
update.update({'isakmp_policy_default_p1_3des': False})
|
|
|
|
# update _default table with device isakmp policy result
|
|
# for i in update['isakmp_policy']:
|
|
# print(i)
|
|
# print(update['isakmp_policy_default_p1_3des'])
|
|
filter = {'DeviceName': device_name}
|
|
result = collection.update_one(filter, {'$set': update}, upsert=True)
|
|
# print(dir(result))
|
|
# print(result.acknowledged) # need to print the full insert update upsert stuff
|
|
return result
|
|
else:
|
|
print(f'isakmp policy entry count {len(isakmp_policy)}')
|
|
|
|
def cisco_transform_set(collection, command_output, device_record, connection):
|
|
print('\ncisco_transform_set')
|
|
|
|
def process_transform_set(scrape):
|
|
# print(scrape)
|
|
tfs_found = False
|
|
tfset = []
|
|
for line in scrape.split('\n'):
|
|
sline = line.lstrip()
|
|
if not tfs_found:
|
|
if 'Transform sets=' in sline:
|
|
tfs_found = True
|
|
pass
|
|
elif ' } ,' in sline:
|
|
tfs_name = sline.split(' ')[0].split(':')[0]
|
|
tfs_encr_algo = sline.replace(' ', ' ').split(' ')[2]
|
|
tfs_hash_algo = sline.replace(' ', ' ').split(' ')[3]
|
|
tfset.append({'name': tfs_name, 'p2_encr_algo': tfs_encr_algo, 'p2_hash_algo': tfs_hash_algo})
|
|
else:
|
|
tfs_found = False
|
|
return tfset
|
|
|
|
## init vars
|
|
update_src = []
|
|
device_name = device_record['DeviceName']
|
|
device_type = device_record['DeviceType']
|
|
device_table = collection['temp'][device_name]
|
|
requests = []
|
|
|
|
## dmvpn lookup ordered transform set
|
|
# print('dmvpn')
|
|
if device_type in ["IP-VPNHUB", "IP-VCSR-HUB"]:
|
|
tunnel_interfaces = device_table.distinct('p2_interface')
|
|
print(f'tunnel interfaces {tunnel_interfaces}')
|
|
if len(tunnel_interfaces) >0:
|
|
for t in tunnel_interfaces:
|
|
interface_name = t
|
|
with Scrapli(**connection) as conn:
|
|
interface = conn.send_command(f'show interface {interface_name}')
|
|
parsed = interface.genie_parse_output()
|
|
# print(json.dumps(parsed, indent=4))
|
|
if 'tunnel_profile' in parsed[interface_name]:
|
|
ipsec_profile_name = parsed[interface_name]['tunnel_profile']
|
|
elif 'Tunnel protection via IPSec' in interface.result:
|
|
# some ios genie outputs are not fully parsed, failback to manual parse
|
|
for line in interface.result.split('\n'):
|
|
if 'Tunnel protection via IPSec' in line:
|
|
ipsec_profile_name = [a for a in line[line.find("(")+1:line.find(")")].split()][1]
|
|
# print(ipsec_profile_name)
|
|
if 'ipsec_profile_name' in locals():
|
|
with Scrapli(**connection) as conn:
|
|
ipsec_profile = conn.send_command(f'show crypto ipsec profile {ipsec_profile_name}')
|
|
# print(ipsec_profile.result)
|
|
transform_set = process_transform_set(ipsec_profile.result)
|
|
match_field = 'p2_interface'
|
|
match_field_value = t
|
|
update_src.append({'match_field': match_field, 'match_field_value': match_field_value, 'transform_set': transform_set})
|
|
|
|
## rri lookup ordered transform set
|
|
# print('rri')
|
|
if device_type in ["IP-VPNAGG", "IP-P2PAGG"]:
|
|
crypto_map_templates = device_table.distinct('crypto_map_template')
|
|
# print(crypto_map_templates)
|
|
if len(crypto_map_templates) >0:
|
|
for t in crypto_map_templates:
|
|
with Scrapli(**connection) as conn:
|
|
crypto_map = conn.send_command(f'show crypto dynamic-map tag {t}')
|
|
# print(crypto_map.result)
|
|
transform_set = process_transform_set(crypto_map.result)
|
|
match_field = 'crypto_map_template'
|
|
match_field_value = t
|
|
update_src.append({'match_field': match_field, 'match_field_value': match_field_value, 'transform_set': transform_set})
|
|
|
|
## build db update requests
|
|
# print(json.dumps(update_src, indent=4))
|
|
if len(update_src) >0:
|
|
for r in update_src:
|
|
query = {r['match_field']: r['match_field_value']}
|
|
# print(query)
|
|
object_ids = [d for d in device_table.distinct('_id', query)]
|
|
# print(object_ids)
|
|
query = { "_id" : { "$in" : object_ids } }
|
|
update = {'ordered_transform_set': r['transform_set']}
|
|
requests.append(UpdateMany(query, {'$set': update}, upsert=True))
|
|
# print(requests)
|
|
|
|
## bulk update collection documents with ordered_transform_sets
|
|
if len(requests) >0:
|
|
dst_result = device_table.bulk_write(requests)
|
|
print(dst_result.bulk_api_result)
|
|
return dst_result
|
|
|
|
def triple_des_check(collection, command_output, device_record, connection):
|
|
print('\ntriple_des_check')
|
|
|
|
## owing to the age of mongodb 3.0.15 some filters/operators are not available, the following queries could otherwise be merged and done in bulk in the query language with a huge performance uptick
|
|
# "$arrayElemAt" "$first" "$slice", "$regex" also does not honour read ahead negative match (?!3des)
|
|
# https://stackoverflow.com/questions/29664097/what-is-the-syntax-for-mongodb-query-for-boolean-values
|
|
# https://www.tutorialspoint.com/get-the-first-element-in-an-array-and-return-using-mongodb-aggregate
|
|
|
|
def p2_encr_algo_check(collection, triple_des_match = True):
|
|
if triple_des_match:
|
|
#regex_statement = {'$regex': '.*3des.*', '$options': 'i'}
|
|
regex_statement = re.compile('(?i).*3DES.*')
|
|
else:
|
|
regex_statement = {'$not': re.compile('(?i).*3DES.*')}
|
|
result = collection.aggregate([
|
|
{"$match": {"ordered_transform_set": {"$exists": True}}},
|
|
{"$match": {'p2_encr_algo': regex_statement }},
|
|
{"$project": {"_id": 1}}
|
|
])
|
|
matched_doc_ids = [d['_id'] for d in result]
|
|
# print(dumps(matched_doc_ids, indent=4))
|
|
return matched_doc_ids
|
|
|
|
def first_ordered_transform_set_check(collection, doc_ids, triple_des_match = True):
|
|
matched_doc_ids = []
|
|
if triple_des_match:
|
|
regex_statement = re.compile('(?i).*3DES.*')
|
|
else:
|
|
regex_statement = {'$not': re.compile('(?i).*3DES.*')}
|
|
for doc_id in doc_ids:
|
|
result = collection.aggregate([
|
|
{"$match": { "_id" : doc_id }},
|
|
{"$unwind": "$ordered_transform_set"},
|
|
{"$limit": 1 },
|
|
{"$match": {'ordered_transform_set.p2_encr_algo': regex_statement}},
|
|
{"$project": {"_id": 1}}
|
|
])
|
|
for result_id in [d['_id'] for d in result]:
|
|
matched_doc_ids.append(result_id)
|
|
# print(dumps(matched_doc_ids, indent=4))
|
|
return matched_doc_ids
|
|
|
|
def tdes_requests_builder(requests, doc_ids, p2_default_3des, spoke_p2_default_3des, spoke_p2_algo_preference):
|
|
if len(doc_ids) >0:
|
|
update = {}
|
|
update.update({'p2_default_3des': p2_default_3des})
|
|
if spoke_p2_default_3des != 'unset':
|
|
update.update({'spoke_p2_default_3des': spoke_p2_default_3des})
|
|
update.update({'spoke_p2_algo_preference': spoke_p2_algo_preference})
|
|
# print(json.dumps(update, indent=4))
|
|
query = { "_id" : { "$in" : doc_ids } }
|
|
requests.append(UpdateMany(query, {'$set': update}, upsert=True))
|
|
return requests
|
|
|
|
## init vars
|
|
device_name = device_record['DeviceName']
|
|
device_table = collection['temp'][device_name]
|
|
requests = []
|
|
|
|
#### p2_encr_algo 3des
|
|
triple_des_match = True
|
|
tdes_doc_ids = p2_encr_algo_check(device_table, triple_des_match)
|
|
|
|
## 1st ordered_transform_set 3des
|
|
triple_des_match = True
|
|
tdes_tdes_ids = first_ordered_transform_set_check(device_table, tdes_doc_ids, triple_des_match)
|
|
# p2_default_3des = True / spoke_p2_default_3des = unset / spoke_p2_algo_preference = unknown
|
|
requests = tdes_requests_builder(requests, tdes_tdes_ids, True, 'unset', 'unknown')
|
|
|
|
## 1st ordered_transform_set NOT 3des
|
|
triple_des_match = False
|
|
tdes_ntdes_ids = first_ordered_transform_set_check(device_table, tdes_doc_ids, triple_des_match)
|
|
# p2_default_3des False / spoke_p2_default_3des True / spoke_p2_algo_preference = 3des
|
|
requests = tdes_requests_builder(requests, tdes_ntdes_ids, False, True, '3des')
|
|
|
|
#### p2_encr_algo NOT 3des
|
|
triple_des_match = False
|
|
ntdes_doc_ids = p2_encr_algo_check(device_table, triple_des_match)
|
|
|
|
## 1st ordered_transform_set 3des
|
|
triple_des_match = True
|
|
ntdes_tdes_ids = first_ordered_transform_set_check(device_table, ntdes_doc_ids, triple_des_match)
|
|
# p2_default_3des True / spoke_p2_default_3des False / spoke_p2_algo_preference = not 3des
|
|
requests = tdes_requests_builder(requests, ntdes_tdes_ids, True, False, 'not 3des')
|
|
|
|
## 1st ordered_transform_set NOT 3des
|
|
triple_des_match = False
|
|
ntdes_ntdes_ids = first_ordered_transform_set_check(device_table, ntdes_doc_ids, triple_des_match)
|
|
# p2_default_3des False / spoke_p2_default_3des unset / spoke_p2_algo_preference = unknown
|
|
requests = tdes_requests_builder(requests, ntdes_ntdes_ids, False, 'unset', 'unknown')
|
|
|
|
## bulk update collection documents with ordered_transform_sets
|
|
if len(requests) >0:
|
|
result = device_table.bulk_write(requests)
|
|
print(result.bulk_api_result)
|
|
return result |