diff --git a/src/components/chart/ha-chart-base.ts b/src/components/chart/ha-chart-base.ts
index 8bda2bfba8..bbf2fcacd7 100644
--- a/src/components/chart/ha-chart-base.ts
+++ b/src/components/chart/ha-chart-base.ts
@@ -108,7 +108,10 @@ export class HaChartBase extends LitElement {
 
     // Add keyboard event listeners
     const handleKeyDown = (ev: KeyboardEvent) => {
-      if ((isMac && ev.key === "Meta") || (!isMac && ev.key === "Control")) {
+      if (
+        !this._modifierPressed &&
+        ((isMac && ev.key === "Meta") || (!isMac && ev.key === "Control"))
+      ) {
         this._modifierPressed = true;
         if (!this.options?.dataZoom) {
           this._setChartOptions({ dataZoom: this._getDataZoomConfig() });
@@ -123,7 +126,10 @@ export class HaChartBase extends LitElement {
     };
 
     const handleKeyUp = (ev: KeyboardEvent) => {
-      if ((isMac && ev.key === "Meta") || (!isMac && ev.key === "Control")) {
+      if (
+        this._modifierPressed &&
+        ((isMac && ev.key === "Meta") || (!isMac && ev.key === "Control"))
+      ) {
         this._modifierPressed = false;
         if (!this.options?.dataZoom) {
           this._setChartOptions({ dataZoom: this._getDataZoomConfig() });