111 lines
5.3 KiB
Python
111 lines
5.3 KiB
Python
import re
|
|
import socket
|
|
from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne, UpdateMany
|
|
import logging
|
|
|
|
def device_record(collection):
|
|
# print('\ndevice_record')
|
|
logger = logging.getLogger('main')
|
|
logger.info('Lookup mongodb for device records')
|
|
|
|
### Query parameters
|
|
## DeviceType
|
|
# exisitng device types ["IP-VPNHUB", "IP-VPNAGG", "IP-P2PAGG"]
|
|
# new type 'IP-VCSR-HUB' for cloud routers
|
|
## DeviceStatus
|
|
# {'Testing', 'Installed', 'Order Cancelled', 'Hold', 'De-Installed', 'Configured', 'Operational', 'Pend De-Install', 'New'}
|
|
# dont exclude 'Operational' / 'Testing' / 'Configured' the device is up and may have tunnels
|
|
# exclude 'New' the device maybe physically installed and not configured
|
|
# exclude 'Pend De-Install' the device doesnt seem to be reachable, maybe powered off ready to de-rack
|
|
## DeviceName
|
|
# exclude devices suffixed _old or _ol
|
|
## Environment_Usage
|
|
# exclude 'QA' / 'UAT', transitory tunnels no need to record
|
|
query = { "raw.DeviceType": {"$in": ["IP-VPNHUB", "IP-VPNAGG", "IP-P2PAGG", "IP-VCSR-HUB"]},
|
|
"raw.Environment_Usage": {"$nin": ["QA", "UAT"]},
|
|
"raw.DeviceStatus": {"$nin": ["Order Cancelled", "De-Installed", "Installed", "New", "Hold", "Pend De-Install"]},
|
|
"raw.DeviceName": {"$nin": [re.compile('.*_old$'), re.compile('.*_ol$')]}
|
|
}
|
|
|
|
result = collection.find(query)
|
|
device_dict = {}
|
|
include_fields = ['DeviceRecNum', 'DeviceType', 'DeviceDescription', 'DeviceStatus', 'Site', 'Country', 'Region', 'Division']
|
|
for i in result:
|
|
device_entry = {}
|
|
device_attributes = {}
|
|
if 'DeviceName' in i['raw']:
|
|
for l in include_fields:
|
|
if l in i['raw']:
|
|
device_attributes.update({l: i['raw'][l]})
|
|
device_entry = { i['raw']['DeviceName']: device_attributes }
|
|
device_dict.update(device_entry)
|
|
return device_dict
|
|
|
|
def mgmt_address(collection, device_dict):
|
|
# print('\nmgmt_address')
|
|
logger = logging.getLogger('main')
|
|
logger.info('Lookup mongodb for device management addresses')
|
|
for device in device_dict.keys():
|
|
query = { "raw.CHR_DeviceName": device }
|
|
result = collection.find(query)
|
|
for r in result:
|
|
# a populated 'DNS_Update_Timestamp' field seems to be the best indicator of the management IP record
|
|
if r['normalized']['DNS_Update_Timestamp'] is not None:
|
|
if r['raw']['CHR_FQDN'] is not None:
|
|
device_dict[device].update({'FQDN': r['raw']['CHR_FQDN']})
|
|
if r['normalized']['CHR_IPAddress']['ip'] is not None:
|
|
device_dict[device].update({'IPAddress': r['normalized']['CHR_IPAddress']['ip']})
|
|
return(device_dict)
|
|
|
|
def dns_lookup(suffix, device_dict):
|
|
# print('\ndns_lookup')
|
|
logger = logging.getLogger('main')
|
|
logger.info('Lookup DNS for absent device management addresses')
|
|
device_lookup = []
|
|
for k in device_dict.keys():
|
|
if 'FQDN' not in device_dict[k]:
|
|
device_lookup.append(k)
|
|
if 'FQDN' in device_dict[k] and not len(device_dict[k]["FQDN"]) >0:
|
|
device_lookup.append(k)
|
|
# lookup device in DNS
|
|
if len(device_lookup) >0:
|
|
# print(f'{len(device_lookup)} devices with no MongoDB field for FQDN, perform DNS lookup for: \n {device_lookup}')
|
|
logger.info(f'{len(device_lookup)} devices with no MongoDB field for FQDN, perform DNS lookup for:')
|
|
logger.info(f'{device_lookup}')
|
|
for name in device_lookup:
|
|
for s in suffix:
|
|
fqdn = name + '.' + s
|
|
try:
|
|
ip = socket.gethostbyname(fqdn)
|
|
# print(f'found DNS record {fqdn}')
|
|
logger.info(f'found DNS record {fqdn}')
|
|
device_dict[name].update({"FQDN": fqdn, "IPAddress": ip})
|
|
except socket.gaierror as e:
|
|
pass
|
|
# populate 'unknown' fields for devices found in MongoDB without DNS records (this is just a catchall to negate further device inspection)
|
|
for k in device_dict.keys():
|
|
if 'FQDN' not in device_dict[k] or not len(device_dict[k]['FQDN']) >0:
|
|
device_dict[k].update({"FQDN": 'unknown'})
|
|
if 'IPAddress' not in device_dict[k] or not len(device_dict[k]['IPAddress']) >0:
|
|
device_dict[k].update({"IPAddress": 'unknown'})
|
|
# print(f'{device_dict["lon-vpn22"]}')
|
|
return(device_dict)
|
|
|
|
def write_devices_collection(collection, device_dict):
|
|
# print('\nwrite_devices_collection')
|
|
logger = logging.getLogger('main')
|
|
logger.info('Write updated device records to collection')
|
|
# merge 'DeviceName' from key into value dict
|
|
records = [{**device_dict[i], 'DeviceName': i} for i in device_dict.keys()]
|
|
requests = []
|
|
for i in records:
|
|
record = i
|
|
filter = {'DeviceName': i['DeviceName']}
|
|
requests.append(ReplaceOne(filter, record, upsert=True))
|
|
result = collection.bulk_write(requests)
|
|
# object_ids = [str(bson.objectid.ObjectId(oid=id)) for id in result.upserted_ids.values()]
|
|
# print(result.bulk_api_result)
|
|
# logger.info(result.bulk_api_result)
|
|
logger.info(f'Database: inserted_count {result.inserted_count} upserted_count {result.upserted_count} matched_count {result.matched_count} modified_count {result.modified_count} deleted_count {result.deleted_count}')
|
|
object_ids = [id for id in result.upserted_ids.values()]
|
|
return object_ids |