diff --git a/docs/release-notes/version-2.9.md b/docs/release-notes/version-2.9.md
index 093dc34a7..6d4f56ed8 100644
--- a/docs/release-notes/version-2.9.md
+++ b/docs/release-notes/version-2.9.md
@@ -9,6 +9,7 @@
 ### Bug Fixes
 
 * [#5050](https://github.com/netbox-community/netbox/issues/5050) - Fix potential failure on `0016_replicate_interfaces` schema migration from old release
+* [#5105](https://github.com/netbox-community/netbox/issues/5105) - Validation should fail when reassigning a primary IP from device to VM
 * [#5109](https://github.com/netbox-community/netbox/issues/5109) - Fix representation of custom choice field values for webhook data
 * [#5108](https://github.com/netbox-community/netbox/issues/5108) - Fix execution of reports via CLI
 * [#5111](https://github.com/netbox-community/netbox/issues/5111) - Allow use of tuples when specifying ObjectVar `query_params`
diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py
index 1e8e9038a..75a4caf10 100644
--- a/netbox/ipam/forms.py
+++ b/netbox/ipam/forms.py
@@ -641,11 +641,11 @@ class IPAddressForm(BootstrapMixin, TenancyForm, ReturnURLForm, CustomFieldModel
                 self.initial['primary_for_parent'] = True
 
     def clean(self):
-        super().clean()
 
         # Cannot select both a device interface and a VM interface
         if self.cleaned_data.get('interface') and self.cleaned_data.get('vminterface'):
             raise forms.ValidationError("Cannot select both a device interface and a virtual machine interface")
+        self.instance.assigned_object = self.cleaned_data.get('interface') or self.cleaned_data.get('vminterface')
 
         # Primary IP assignment is only available if an interface has been assigned.
         interface = self.cleaned_data.get('interface') or self.cleaned_data.get('vminterface')
@@ -655,26 +655,21 @@ class IPAddressForm(BootstrapMixin, TenancyForm, ReturnURLForm, CustomFieldModel
             )
 
     def save(self, *args, **kwargs):
-
-        # Set assigned object
-        interface = self.cleaned_data.get('interface') or self.cleaned_data.get('vminterface')
-        if interface:
-            self.instance.assigned_object = interface
-
         ipaddress = super().save(*args, **kwargs)
 
         # Assign/clear this IPAddress as the primary for the associated Device/VirtualMachine.
+        interface = self.instance.assigned_object
         if interface and self.cleaned_data['primary_for_parent']:
             if ipaddress.address.version == 4:
                 interface.parent.primary_ip4 = ipaddress
             else:
-                interface.primary_ip6 = ipaddress
+                interface.parent.primary_ip6 = ipaddress
             interface.parent.save()
         elif interface and ipaddress.address.version == 4 and interface.parent.primary_ip4 == ipaddress:
             interface.parent.primary_ip4 = None
             interface.parent.save()
         elif interface and ipaddress.address.version == 6 and interface.parent.primary_ip6 == ipaddress:
-            interface.parent.primary_ip4 = None
+            interface.parent.primary_ip6 = None
             interface.parent.save()
 
         return ipaddress
diff --git a/netbox/ipam/models.py b/netbox/ipam/models.py
index 832e09330..c7955bcfa 100644
--- a/netbox/ipam/models.py
+++ b/netbox/ipam/models.py
@@ -726,30 +726,18 @@ class IPAddress(ChangeLoggedModel, CustomFieldModel):
                     })
 
         # Check for primary IP assignment that doesn't match the assigned device/VM
-        if self.pk and type(self.assigned_object) is Interface:
+        if self.pk:
             device = Device.objects.filter(Q(primary_ip4=self) | Q(primary_ip6=self)).first()
             if device:
-                if self.assigned_object is None:
+                if getattr(self.assigned_object, 'device', None) != device:
                     raise ValidationError({
-                        'interface': f"IP address is primary for device {device} but not assigned to an interface"
+                        'interface': f"IP address is primary for device {device} but not assigned to it!"
                     })
-                elif self.assigned_object.device != device:
-                    raise ValidationError({
-                        'interface': f"IP address is primary for device {device} but assigned to "
-                                     f"{self.assigned_object.device} ({self.assigned_object})"
-                    })
-        elif self.pk and type(self.assigned_object) is VMInterface:
             vm = VirtualMachine.objects.filter(Q(primary_ip4=self) | Q(primary_ip6=self)).first()
             if vm:
-                if self.assigned_object is None:
+                if getattr(self.assigned_object, 'virtual_machine', None) != vm:
                     raise ValidationError({
-                        'vminterface': f"IP address is primary for virtual machine {vm} but not assigned to an "
-                                       f"interface"
-                    })
-                elif self.assigned_object.virtual_machine != vm:
-                    raise ValidationError({
-                        'vminterface': f"IP address is primary for virtual machine {vm} but assigned to "
-                                       f"{self.assigned_object.virtual_machine} ({self.assigned_object})"
+                        'vminterface': f"IP address is primary for virtual machine {vm} but not assigned to it!"
                     })
 
         # Validate IP status selection