diff --git a/src/components/entity/ha-entity-state-content-picker.ts b/src/components/entity/ha-entity-state-content-picker.ts
index 2ccc606668..2e1e25c988 100644
--- a/src/components/entity/ha-entity-state-content-picker.ts
+++ b/src/components/entity/ha-entity-state-content-picker.ts
@@ -178,7 +178,7 @@ class HaEntityStatePicker extends LitElement {
               no-style
               @item-moved=${this._moveItem}
               .disabled=${this.disabled}
-              handle-selector="[data-handle]"
+              handle-selector="button.primary.action"
             >
               <ha-chip-set>
                 ${repeat(
@@ -194,7 +194,6 @@ class HaEntityStatePicker extends LitElement {
                         @remove=${this._removeItem}
                         .label=${label}
                         selected
-                        data-handle
                       >
                         <ha-svg-icon slot="icon" .path=${mdiDrag}></ha-svg-icon>
                         ${label}
diff --git a/src/components/ha-selector/ha-selector-select.ts b/src/components/ha-selector/ha-selector-select.ts
index bb5fb989ef..7e0f9ed52d 100644
--- a/src/components/ha-selector/ha-selector-select.ts
+++ b/src/components/ha-selector/ha-selector-select.ts
@@ -156,7 +156,7 @@ export class HaSelectSelector extends LitElement {
                 no-style
                 .disabled=${!this.selector.select.reorder}
                 @item-moved=${this._itemMoved}
-                handle-selector="[data-handle]"
+                handle-selector="button.primary.action"
               >
                 <ha-chip-set>
                   ${repeat(
@@ -172,7 +172,6 @@ export class HaSelectSelector extends LitElement {
                           @remove=${this._removeItem}
                           .label=${label}
                           selected
-                          data-handle
                         >
                           ${this.selector.select?.reorder
                             ? html`