diff --git a/src/panels/lovelace/common/directives/action-handler-directive.ts b/src/panels/lovelace/common/directives/action-handler-directive.ts
index 1a837b5743..09dc3c4351 100644
--- a/src/panels/lovelace/common/directives/action-handler-directive.ts
+++ b/src/panels/lovelace/common/directives/action-handler-directive.ts
@@ -150,13 +150,13 @@ class ActionHandler extends HTMLElement implements ActionHandler {
       }
       // Prevent mouse event if touch event
       ev.preventDefault();
-      if (
-        ["touchend", "touchcancel"].includes(ev.type) &&
-        this.timer === undefined
-      ) {
-        return;
-      }
       if (options.hasHold) {
+        if (
+          ["touchend", "touchcancel"].includes(ev.type) &&
+          this.timer === undefined
+        ) {
+          return;
+        }
         clearTimeout(this.timer);
         this.stopAnimation();
         this.timer = undefined;