From b080e4447d2393b7e122bebcb9ef9a202084b44d Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 12:17:22 +1000 Subject: [PATCH 01/19] stops ip addresses assigned to FHRP groups from being reassigned --- module/netbox/object_classes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 9d25093..f032383 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2152,6 +2152,10 @@ def update(self, data=None, read_from_netbox=False, source=None): object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") + if object_type == "ipam.fhrpgroup": + log.info("IP address assigned to FHRP group. Skipping.") + return + # used to track changes in object primary IP assignments previous_ip_device_vm = None is_primary_ipv4_of_previous_device = False From 049655a8001f72eb223bcfe850a5e7a8ff413593 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 13:24:10 +1000 Subject: [PATCH 02/19] worded the log better --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index f032383..194b625 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2153,7 +2153,7 @@ def update(self, data=None, read_from_netbox=False, source=None): assigned_object = data.get("assigned_object_id") if object_type == "ipam.fhrpgroup": - log.info("IP address assigned to FHRP group. Skipping.") + log.info(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") return # used to track changes in object primary IP assignments From 5f16691744c31eccafff9df9731b6b1bdb44093b Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 15:42:31 +1000 Subject: [PATCH 03/19] added skipping_fhrp_group_ips config option to enable or disable fhrp group ip address overriding --- module/netbox/object_classes.py | 14 ++++++++++---- module/sources/check_redfish/config.py | 6 ++++++ module/sources/vmware/config.py | 5 +++++ module/sources/vmware/connection.py | 6 ++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 194b625..dfa49f5 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2148,12 +2148,18 @@ def resolve_relations(self): super().resolve_relations() def update(self, data=None, read_from_netbox=False, source=None): - object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") - - if object_type == "ipam.fhrpgroup": - log.info(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + + # Skip IP assignments when the IP is assigned to FHRP groups when config option + # skipping_fhrp_group_ips is set to True + if source is not None: + config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") + if config_relation == True and object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return + elif object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") return # used to track changes in object primary IP assignments diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 2c21fff..2446671 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -70,6 +70,12 @@ def __init__(self): via check_redfish if False only data which is not preset in NetBox will be added""", default_value=False), + ConfigOption("skipping_fhrp_group_ips", + bool, + description="""define if an IP address assigned to a FHRP group (like HSRP, VRRP, GLBP) will be skipped. + If True this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", + default_value=False), + ConfigOption(**config_option_ip_tenant_inheritance_order_definition), ] diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 7e448a2..14acdec 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -258,6 +258,11 @@ def __init__(self): description="""If the VMware Site Recovery Manager is used to can skip syncing placeholder/replicated VMs from fail-over site to NetBox.""", default_value=False), + ConfigOption("skipping_fhrp_group_ips", + bool, + description="""If an IP address is assigned to a FHRP group (like HSRP, VRRP, GLBP) + then this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", + default_value=False), ConfigOption("strip_host_domain_name", bool, description="strip domain part from host name before syncing device to NetBox", diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 742f54c..d275742 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -914,6 +914,12 @@ def get_object_relation(self, name, relation, fallback=None): """ resolved_list = list() + relation_data = grab(self.settings, relation, fallback=fallback) + + if isinstance(relation_data, bool): + log.debug(f"Object relation '{relation}' is boolean '{relation_data}'. Returning.") + return relation_data + for single_relation in grab(self.settings, relation, fallback=list()): object_regex = single_relation.get("object_regex") match_found = False From 960680c47ead2cd9292c5e9fb69fccfacee37eae Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 16:23:45 +1000 Subject: [PATCH 04/19] limited skipping_fhrp_group_ips to vmware since I can't test redfish presently --- module/netbox/object_classes.py | 11 ++++++----- module/sources/vmware/connection.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dfa49f5..3f0397b 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2154,12 +2154,13 @@ def update(self, data=None, read_from_netbox=False, source=None): # Skip IP assignments when the IP is assigned to FHRP groups when config option # skipping_fhrp_group_ips is set to True if source is not None: - config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return + if source.source_type == "vmware": + config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") + if config_relation == True and object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") return # used to track changes in object primary IP assignments diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index d275742..542f08a 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -917,7 +917,7 @@ def get_object_relation(self, name, relation, fallback=None): relation_data = grab(self.settings, relation, fallback=fallback) if isinstance(relation_data, bool): - log.debug(f"Object relation '{relation}' is boolean '{relation_data}'. Returning.") + log.debug(f"Object relation '{relation}' is boolean, set '{relation_data}'.") return relation_data for single_relation in grab(self.settings, relation, fallback=list()): From dae2911659fca4df8967222dbe6b9b16da42ac31 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 16:27:09 +1000 Subject: [PATCH 05/19] tweaked comment --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 3f0397b..455a833 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2152,7 +2152,7 @@ def update(self, data=None, read_from_netbox=False, source=None): assigned_object = data.get("assigned_object_id") # Skip IP assignments when the IP is assigned to FHRP groups when config option - # skipping_fhrp_group_ips is set to True + # skipping_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) if source is not None: if source.source_type == "vmware": config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") From b61a34bbc979e98cfa29d1c615521a6829f60164 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 09:54:04 +1000 Subject: [PATCH 06/19] limited skip_fhrp_group_ips to vmware since I can't test redfish presently --- module/netbox/object_classes.py | 13 +++++++------ module/sources/check_redfish/config.py | 2 +- module/sources/vmware/config.py | 2 +- module/sources/vmware/connection.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dfa49f5..0b954a6 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2152,14 +2152,15 @@ def update(self, data=None, read_from_netbox=False, source=None): assigned_object = data.get("assigned_object_id") # Skip IP assignments when the IP is assigned to FHRP groups when config option - # skipping_fhrp_group_ips is set to True + # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) if source is not None: - config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return + if source.source_type == "vmware": + config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") + if config_relation == True and object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") return # used to track changes in object primary IP assignments diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 2446671..a4f99be 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -70,7 +70,7 @@ def __init__(self): via check_redfish if False only data which is not preset in NetBox will be added""", default_value=False), - ConfigOption("skipping_fhrp_group_ips", + ConfigOption("skip_fhrp_group_ips", bool, description="""define if an IP address assigned to a FHRP group (like HSRP, VRRP, GLBP) will be skipped. If True this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 14acdec..3e5c731 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -258,7 +258,7 @@ def __init__(self): description="""If the VMware Site Recovery Manager is used to can skip syncing placeholder/replicated VMs from fail-over site to NetBox.""", default_value=False), - ConfigOption("skipping_fhrp_group_ips", + ConfigOption("skip_fhrp_group_ips", bool, description="""If an IP address is assigned to a FHRP group (like HSRP, VRRP, GLBP) then this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index d275742..542f08a 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -917,7 +917,7 @@ def get_object_relation(self, name, relation, fallback=None): relation_data = grab(self.settings, relation, fallback=fallback) if isinstance(relation_data, bool): - log.debug(f"Object relation '{relation}' is boolean '{relation_data}'. Returning.") + log.debug(f"Object relation '{relation}' is boolean, set '{relation_data}'.") return relation_data for single_relation in grab(self.settings, relation, fallback=list()): From 7e0fde97d45bafa41d3f222ecbff479fcbc39cc1 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 13:09:15 +1000 Subject: [PATCH 07/19] modified: module/sources/vmware/connection.py --- module/sources/vmware/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 542f08a..a493d37 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -519,6 +519,7 @@ def get_object_based_on_macs(self, object_type, mac_list=None): Returns ------- (NBDevice, NBVM, None): object instance of found device, otherwise None + comment line """ object_to_return = None From ee80b802d9d88f98ce3f5540dede5448d76e3974 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 13:12:13 +1000 Subject: [PATCH 08/19] modified: module/sources/vmware/connection.py --- module/sources/vmware/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index a493d37..542f08a 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -519,7 +519,6 @@ def get_object_based_on_macs(self, object_type, mac_list=None): Returns ------- (NBDevice, NBVM, None): object instance of found device, otherwise None - comment line """ object_to_return = None From 0be06699759a27faa9d1f10c4b9a4b7f2fe621c9 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 16:53:44 +1000 Subject: [PATCH 09/19] changed the NBIPAddress data model entry 'assigned_object_type' to have set valid types --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 0b954a6..999c852 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2131,7 +2131,7 @@ def __init__(self, *args, **kwargs): ] self.data_model = { "address": str, - "assigned_object_type": self.mapping.scopes_object_types(self.scopes), + "assigned_object_type": ("ipam.fhrpgroup, dcim.interface, virtualization.vminterface"), "assigned_object_id": self.scopes, "description": 200, "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], From 2e3fb455f3a4c89508738281125a614bff4191fe Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 27 Aug 2025 09:34:14 +1000 Subject: [PATCH 10/19] modified: module/netbox/object_classes.py --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 999c852..0b954a6 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2131,7 +2131,7 @@ def __init__(self, *args, **kwargs): ] self.data_model = { "address": str, - "assigned_object_type": ("ipam.fhrpgroup, dcim.interface, virtualization.vminterface"), + "assigned_object_type": self.mapping.scopes_object_types(self.scopes), "assigned_object_id": self.scopes, "description": 200, "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], From 6ffa661705e39c5590bc787ea4a39a30443d4fe9 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 27 Aug 2025 16:54:30 +1000 Subject: [PATCH 11/19] modified: module/netbox/object_classes.py --- module/netbox/object_classes.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 0b954a6..b88efb2 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2150,18 +2150,6 @@ def resolve_relations(self): def update(self, data=None, read_from_netbox=False, source=None): object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") - - # Skip IP assignments when the IP is assigned to FHRP groups when config option - # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) - if source is not None: - if source.source_type == "vmware": - config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return - elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") - return # used to track changes in object primary IP assignments previous_ip_device_vm = None @@ -2173,6 +2161,19 @@ def update(self, data=None, read_from_netbox=False, source=None): # get current device to make sure to unset primary ip before moving IP address previous_ip_device_vm = self.get_device_vm() + + # Skip IP assignments when the IP is already assigned to FHRP groups when config option + # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) + if source is not None: + if source.source_type == "vmware": + config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") + if config_relation == True and grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return + elif grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") + return + if grab(previous_ip_device_vm, "data.primary_ip4") is self: is_primary_ipv4_of_previous_device = True if grab(previous_ip_device_vm, "data.primary_ip6") is self: @@ -2213,7 +2214,7 @@ def get_interface(self): o_id = self.data.get("assigned_object_id") o_type = self.data.get("assigned_object_type") - if isinstance(o_id, (NBInterface, NBVMInterface)): + if isinstance(o_id, (NBInterface, NBVMInterface, NBFHRPGroupItem)): return o_id if o_type is None or not isinstance(o_id, int): @@ -2235,6 +2236,8 @@ def get_device_vm(self): return o_interface.data.get("device") elif isinstance(o_interface, NBVMInterface): return o_interface.data.get("virtual_machine") + elif isinstance(o_interface, NBFHRPGroupItem): + return o_interface.data.get("fhrp_group") def remove_interface_association(self): o_id = self.data.get("assigned_object_id") From 3a22fc2e743a2bb2397e021403ddc013152e1b53 Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 28 Aug 2025 10:30:47 +1000 Subject: [PATCH 12/19] Undid part of a previous commit, to undo a mistake --- module/netbox/object_classes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index b88efb2..45530a2 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2167,10 +2167,10 @@ def update(self, data=None, read_from_netbox=False, source=None): if source is not None: if source.source_type == "vmware": config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") - if config_relation == True and grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + if config_relation == True and object_type == "ipam.fhrpgroup": log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") return - elif grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + elif object_type == "ipam.fhrpgroup": log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") return From 3d9a189fa0fb8383116761c863a49678444dcb9a Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 28 Aug 2025 15:56:12 +1000 Subject: [PATCH 13/19] Added a dev document for explaining how the code works for the benefit of other devs --- docs/dev-sync-process.md | 76 +++++++++++++++++++++++++++++ module/sources/vmware/connection.py | 4 +- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 docs/dev-sync-process.md diff --git a/docs/dev-sync-process.md b/docs/dev-sync-process.md new file mode 100644 index 0000000..9ba4d18 --- /dev/null +++ b/docs/dev-sync-process.md @@ -0,0 +1,76 @@ +This document is intended to be an overview of how the code is working. + +1. source list is validated. +2. sources are instantiated. + the sources are returned as a source handler list. +3. netbox data is queried and cached + All data types in NBInventory class are: + "FHRP group": [], + "IP address": [], + "IP prefix": [], + "MAC address": [], + "VLAN": [], + "VLANGroup": [], + "VRF": [], + "Virtual Disk": [], + "cluster": [], + "cluster group": [], + "cluster type": [], + "custom field": [], + "device": [], + "device role": [], + "device type": [], + "interface": [], + "inventory item": [], + "manufacturer": [], + "platform": [], + "power port": [], + "site": [], + "site group": [], + "tag": [], + "tenant": [], + "virtual machine": [], + "virtual machine interface": [] + these are all of the possible, valid inventory entries. obtained from adding 'log.debug(inventory)' to module/sources/__init__.py, line 84. +4. source handler 'apply()' method is called, which retrieves vmware data. In this method, 'view handers' are called, which individually add each data type (e.g. data center, cluster) to the 'object_mapping' dict. + object_mapping entries are: + "datacenter": { + "view_type": vim.Datacenter, + "view_handler": self.add_datacenter + }, + "cluster": { + "view_type": vim.ClusterComputeResource, + "view_handler": self.add_cluster + }, + "single host cluster": { + "view_type": vim.ComputeResource, + "view_handler": self.add_cluster + }, + "network": { + "view_type": vim.dvs.DistributedVirtualPortgroup, + "view_handler": self.add_port_group + }, + "host": { + "view_type": vim.HostSystem, + "view_handler": self.add_host + }, + "virtual machine": { + "view_type": vim.VirtualMachine, + "view_handler": self.add_virtual_machine + }, + "offline virtual machine": { + "view_type": vim.VirtualMachine, + "view_handler": self.add_virtual_machine + } +5. the queried data is added to the cache under the view handler methods. + the view handler methods are responsible for adding all the individual data to the cache and inventory +6. in the vm view handler, all the extra data from the vm's is collected and parsed, and then passed into the add_device_vm_to_inventory method. currently i believe this is where the ip address data is coming from. the params for the above method are: + NBVM, object_data=vm_data, vnic_data=nic_data, nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6, vmware_object=obj, disk_data=disk_data +7. in this file /module/sources/vmware/connection.py, line 956 is the method add_device_vm_to_inventory. this is where vm's are added to the inventory, and the corresponding objects (vm attributes, like site, device, primary ip address) are matched or created. +8. after looking through the add_update_interface method (/module/sources/common/source_base.py, line 234) within the previous method, vmware does not appear to be providing any fhrp group related data. itRequest logs is possible this is because i don't have any fhrp group data to sync. + interface data provided (in the form ([ip address], {assigned object})): + ([], {'name': 'vmnic0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'description': '1Gb/s pNIC (vSwitch0)', 'type': '1000base-t', 'mtu': 1500, 'speed': 1000000, 'duplex': 'full', 'mode': 'access'}) + (['192.168.11.143/255.255.255.0'], {'name': 'vmk0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'mtu': 1500, 'type': 'virtual', 'mode': 'access', 'description': 'Management Network (vSwitch0, vlan ID: 0)'}) + (['192.168.11.147/24'], {'name': 'vNIC 1 (VM Network)', 'virtual_machine': None, 'mac_address': '00:0C:29:F3:3C:69', 'description': 'Network adapter 1 (VirtualVmxnet3) (vlan ID: 0)', 'enabled': True, 'mtu': 1500, 'mode': 'access'}) + this method is where ip address objects are created and added to the inventory if new and updated if not. + the method also returns the ip addresses as a list (as well as the interface object) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 542f08a..553a724 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -909,8 +909,8 @@ def get_object_relation(self, name, relation, fallback=None): Returns ------- - data: str, list, None - string of matching relation or list of matching tags + data: str, list, bool, None + string of matching relation or list of matching tags, or boolean if relation is boolean """ resolved_list = list() From 3f5af8c8c9a6b00564347d24f9091fb22462007e Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 28 Aug 2025 16:59:54 +1000 Subject: [PATCH 14/19] Added/changed where the fhrp group ip address updating prevention occured to prevent a duplicate ip address from being created, causing errors. --- module/netbox/object_classes.py | 12 ------------ module/sources/common/source_base.py | 6 ++++++ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 45530a2..ed0add6 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2162,18 +2162,6 @@ def update(self, data=None, read_from_netbox=False, source=None): # get current device to make sure to unset primary ip before moving IP address previous_ip_device_vm = self.get_device_vm() - # Skip IP assignments when the IP is already assigned to FHRP groups when config option - # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) - if source is not None: - if source.source_type == "vmware": - config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return - elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") - return - if grab(previous_ip_device_vm, "data.primary_ip4") is self: is_primary_ipv4_of_previous_device = True if grab(previous_ip_device_vm, "data.primary_ip6") is self: diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 0e95e5c..7611668 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -443,6 +443,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = None skip_this_ip = False for ip in self.inventory.get_all_items(NBIPAddress): + # stops fhrp group assigned ip addresses from being overridden and assigned to another object type + # if the config skip_fhrp_group_ips is set to True + if grab(ip, "data.assigned_object_type", fallback="") == "ipam.fhrpgroup" and self.settings.skip_fhrp_group_ips: + log.info(f"Ip address {grab(ip, "data.address")} is assigned to an FHRP Group and skip_fhrp_group_ips is set to {self.settings.skip_fhrp_group_ips}, skipping.") + skip_this_ip = True + continue # check if address matches (without prefix length) ip_address_string = grab(ip, "data.address", fallback="") From a7acdac1b7aaedb90cf82cb9452b31bf54777534 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 09:05:14 +1000 Subject: [PATCH 15/19] removed the dev doc since it's not related to this bug --- docs/dev-sync-process.md | 76 ---------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 docs/dev-sync-process.md diff --git a/docs/dev-sync-process.md b/docs/dev-sync-process.md deleted file mode 100644 index 9ba4d18..0000000 --- a/docs/dev-sync-process.md +++ /dev/null @@ -1,76 +0,0 @@ -This document is intended to be an overview of how the code is working. - -1. source list is validated. -2. sources are instantiated. - the sources are returned as a source handler list. -3. netbox data is queried and cached - All data types in NBInventory class are: - "FHRP group": [], - "IP address": [], - "IP prefix": [], - "MAC address": [], - "VLAN": [], - "VLANGroup": [], - "VRF": [], - "Virtual Disk": [], - "cluster": [], - "cluster group": [], - "cluster type": [], - "custom field": [], - "device": [], - "device role": [], - "device type": [], - "interface": [], - "inventory item": [], - "manufacturer": [], - "platform": [], - "power port": [], - "site": [], - "site group": [], - "tag": [], - "tenant": [], - "virtual machine": [], - "virtual machine interface": [] - these are all of the possible, valid inventory entries. obtained from adding 'log.debug(inventory)' to module/sources/__init__.py, line 84. -4. source handler 'apply()' method is called, which retrieves vmware data. In this method, 'view handers' are called, which individually add each data type (e.g. data center, cluster) to the 'object_mapping' dict. - object_mapping entries are: - "datacenter": { - "view_type": vim.Datacenter, - "view_handler": self.add_datacenter - }, - "cluster": { - "view_type": vim.ClusterComputeResource, - "view_handler": self.add_cluster - }, - "single host cluster": { - "view_type": vim.ComputeResource, - "view_handler": self.add_cluster - }, - "network": { - "view_type": vim.dvs.DistributedVirtualPortgroup, - "view_handler": self.add_port_group - }, - "host": { - "view_type": vim.HostSystem, - "view_handler": self.add_host - }, - "virtual machine": { - "view_type": vim.VirtualMachine, - "view_handler": self.add_virtual_machine - }, - "offline virtual machine": { - "view_type": vim.VirtualMachine, - "view_handler": self.add_virtual_machine - } -5. the queried data is added to the cache under the view handler methods. - the view handler methods are responsible for adding all the individual data to the cache and inventory -6. in the vm view handler, all the extra data from the vm's is collected and parsed, and then passed into the add_device_vm_to_inventory method. currently i believe this is where the ip address data is coming from. the params for the above method are: - NBVM, object_data=vm_data, vnic_data=nic_data, nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6, vmware_object=obj, disk_data=disk_data -7. in this file /module/sources/vmware/connection.py, line 956 is the method add_device_vm_to_inventory. this is where vm's are added to the inventory, and the corresponding objects (vm attributes, like site, device, primary ip address) are matched or created. -8. after looking through the add_update_interface method (/module/sources/common/source_base.py, line 234) within the previous method, vmware does not appear to be providing any fhrp group related data. itRequest logs is possible this is because i don't have any fhrp group data to sync. - interface data provided (in the form ([ip address], {assigned object})): - ([], {'name': 'vmnic0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'description': '1Gb/s pNIC (vSwitch0)', 'type': '1000base-t', 'mtu': 1500, 'speed': 1000000, 'duplex': 'full', 'mode': 'access'}) - (['192.168.11.143/255.255.255.0'], {'name': 'vmk0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'mtu': 1500, 'type': 'virtual', 'mode': 'access', 'description': 'Management Network (vSwitch0, vlan ID: 0)'}) - (['192.168.11.147/24'], {'name': 'vNIC 1 (VM Network)', 'virtual_machine': None, 'mac_address': '00:0C:29:F3:3C:69', 'description': 'Network adapter 1 (VirtualVmxnet3) (vlan ID: 0)', 'enabled': True, 'mtu': 1500, 'mode': 'access'}) - this method is where ip address objects are created and added to the inventory if new and updated if not. - the method also returns the ip addresses as a list (as well as the interface object) From 11ed756709bfba7e2d1ac78ecb5abdbb4e756426 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 09:22:16 +1000 Subject: [PATCH 16/19] Removed the last of the unnecessary things (hopefully) --- module/netbox/object_classes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index ed0add6..29f72a6 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2148,6 +2148,7 @@ def resolve_relations(self): super().resolve_relations() def update(self, data=None, read_from_netbox=False, source=None): + object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") @@ -2202,7 +2203,7 @@ def get_interface(self): o_id = self.data.get("assigned_object_id") o_type = self.data.get("assigned_object_type") - if isinstance(o_id, (NBInterface, NBVMInterface, NBFHRPGroupItem)): + if isinstance(o_id, (NBInterface, NBVMInterface)): return o_id if o_type is None or not isinstance(o_id, int): @@ -2224,8 +2225,6 @@ def get_device_vm(self): return o_interface.data.get("device") elif isinstance(o_interface, NBVMInterface): return o_interface.data.get("virtual_machine") - elif isinstance(o_interface, NBFHRPGroupItem): - return o_interface.data.get("fhrp_group") def remove_interface_association(self): o_id = self.data.get("assigned_object_id") From bd3d882f43cebf0ee54bded7e193e2d1ad2c71fb Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 09:25:23 +1000 Subject: [PATCH 17/19] one last tiny thing --- module/netbox/object_classes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 29f72a6..9d25093 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2162,7 +2162,6 @@ def update(self, data=None, read_from_netbox=False, source=None): # get current device to make sure to unset primary ip before moving IP address previous_ip_device_vm = self.get_device_vm() - if grab(previous_ip_device_vm, "data.primary_ip4") is self: is_primary_ipv4_of_previous_device = True if grab(previous_ip_device_vm, "data.primary_ip6") is self: From 409642d5a28a53392faf1fce30409cd260491844 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 13:33:39 +1000 Subject: [PATCH 18/19] cleanup unnecessary code --- module/sources/vmware/connection.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 553a724..742f54c 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -909,17 +909,11 @@ def get_object_relation(self, name, relation, fallback=None): Returns ------- - data: str, list, bool, None - string of matching relation or list of matching tags, or boolean if relation is boolean + data: str, list, None + string of matching relation or list of matching tags """ resolved_list = list() - relation_data = grab(self.settings, relation, fallback=fallback) - - if isinstance(relation_data, bool): - log.debug(f"Object relation '{relation}' is boolean, set '{relation_data}'.") - return relation_data - for single_relation in grab(self.settings, relation, fallback=list()): object_regex = single_relation.get("object_regex") match_found = False From 1e865b7e7a446dfaf5fc7d6fc85d806bd41800c8 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 13:45:58 +1000 Subject: [PATCH 19/19] removed the readme note not in the dev branch --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index b944ac2..0430a67 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ # NetBox-Sync -> [!CAUTION] -> **Maintainer wanted - sunsetting this repository by 31.10.2025 [#474](https://github.com/bb-Ricardo/netbox-sync/issues/474)** - This is a tool to sync data from different sources to a NetBox instance. Available source types: