diff --git a/src/panels/lovelace/cards/hui-button-card.ts b/src/panels/lovelace/cards/hui-button-card.ts
index 58d28bc46a..bb6d54c8f8 100644
--- a/src/panels/lovelace/cards/hui-button-card.ts
+++ b/src/panels/lovelace/cards/hui-button-card.ts
@@ -166,7 +166,6 @@ export class HuiButtonCard extends LitElement implements LovelaceCard {
         tabindex=${ifDefined(
           hasAction(this._config.tap_action) ? "0" : undefined
         )}
-        @keydown=${this._handleKeyDown}
       >
         ${this._config.show_icon
           ? html`
@@ -234,12 +233,6 @@ export class HuiButtonCard extends LitElement implements LovelaceCard {
     return this._ripple;
   });
 
-  private _handleKeyDown(ev: KeyboardEvent) {
-    if (ev.key === "Enter" || ev.key === " ") {
-      handleAction(this, this.hass!, this._config!, "tap");
-    }
-  }
-
   @eventOptions({ passive: true })
   private handleRippleActivate(evt?: Event) {
     this._rippleHandlers.startPress(evt);
diff --git a/src/panels/lovelace/common/directives/action-handler-directive.ts b/src/panels/lovelace/common/directives/action-handler-directive.ts
index bdcbb7b116..9fcb5dc30c 100644
--- a/src/panels/lovelace/common/directives/action-handler-directive.ts
+++ b/src/panels/lovelace/common/directives/action-handler-directive.ts
@@ -30,7 +30,7 @@ interface ActionHandlerElement extends HTMLElement {
     options: ActionHandlerOptions;
     start?: (ev: Event) => void;
     end?: (ev: Event) => void;
-    handleEnter?: (ev: KeyboardEvent) => void;
+    handleKeyDown?: (ev: KeyboardEvent) => void;
   };
 }
 
@@ -117,7 +117,10 @@ class ActionHandler extends HTMLElement implements ActionHandler {
       element.removeEventListener("mousedown", element.actionHandler.start!);
       element.removeEventListener("click", element.actionHandler.end!);
 
-      element.removeEventListener("keyup", element.actionHandler.handleEnter!);
+      element.removeEventListener(
+        "keydown",
+        element.actionHandler.handleKeyDown!
+      );
     } else {
       element.addEventListener("contextmenu", (ev: Event) => {
         const e = ev || window.event;
@@ -196,8 +199,8 @@ class ActionHandler extends HTMLElement implements ActionHandler {
       }
     };
 
-    element.actionHandler.handleEnter = (ev: KeyboardEvent) => {
-      if (ev.keyCode !== 13) {
+    element.actionHandler.handleKeyDown = (ev: KeyboardEvent) => {
+      if (!["Enter", " "].includes(ev.key)) {
         return;
       }
       (ev.currentTarget as ActionHandlerElement).actionHandler!.end!(ev);
@@ -214,7 +217,7 @@ class ActionHandler extends HTMLElement implements ActionHandler {
     });
     element.addEventListener("click", element.actionHandler.end);
 
-    element.addEventListener("keyup", element.actionHandler.handleEnter);
+    element.addEventListener("keydown", element.actionHandler.handleKeyDown);
   }
 
   private startAnimation(x: number, y: number) {