diff --git a/homeassistant/components/modem_callerid/sensor.py b/homeassistant/components/modem_callerid/sensor.py
index 6c08ea8d6cf..469e8ecb994 100644
--- a/homeassistant/components/modem_callerid/sensor.py
+++ b/homeassistant/components/modem_callerid/sensor.py
@@ -106,18 +106,13 @@ class ModemCalleridSensor(SensorEntity):
     @callback
     def _async_incoming_call(self, new_state) -> None:
         """Handle new states."""
-        if new_state == PhoneModem.STATE_RING:
-            if self.native_value == PhoneModem.STATE_IDLE:
-                self._attr_extra_state_attributes = {
-                    CID.CID_NUMBER: "",
-                    CID.CID_NAME: "",
-                }
-        elif new_state == PhoneModem.STATE_CALLERID:
-            self._attr_extra_state_attributes = {
-                CID.CID_NUMBER: self.api.cid_number,
-                CID.CID_NAME: self.api.cid_name,
-            }
-        self._attr_extra_state_attributes[CID.CID_TIME] = self.api.cid_time
+        self._attr_extra_state_attributes = {}
+        if self.api.cid_name:
+            self._attr_extra_state_attributes[CID.CID_NAME] = self.api.cid_name
+        if self.api.cid_number:
+            self._attr_extra_state_attributes[CID.CID_NUMBER] = self.api.cid_number
+        if self.api.cid_time:
+            self._attr_extra_state_attributes[CID.CID_TIME] = self.api.cid_time
         self._attr_native_value = self.api.state
         self.async_write_ha_state()