diff --git a/src/com/android/launcher3/config/FeatureFlags.java b/src/com/android/launcher3/config/FeatureFlags.java
index 9484fffef9..3e59e2b25d 100644
--- a/src/com/android/launcher3/config/FeatureFlags.java
+++ b/src/com/android/launcher3/config/FeatureFlags.java
@@ -17,6 +17,7 @@
 package com.android.launcher3.config;
 
 import android.content.Context;
+import android.content.SharedPreferences;
 
 import com.android.launcher3.BuildConfig;
 import com.android.launcher3.Utilities;
@@ -343,7 +344,15 @@ public final class FeatureFlags {
             for (DebugFlag flag : sDebugFlags) {
                 flag.initialize(context);
             }
-            sDebugFlags.sort((f1, f2) -> f1.key.compareToIgnoreCase(f2.key));
+
+            sDebugFlags.sort((f1, f2) -> {
+                // Sort first by any prefs that the user has changed, then alphabetically.
+                int changeComparison = Boolean.compare(f2.mHasBeenChangedAtLeastOnce,
+                        f1.mHasBeenChangedAtLeastOnce);
+                return changeComparison != 0
+                        ? changeComparison
+                        : f1.key.compareToIgnoreCase(f2.key);
+            });
         }
     }
 
@@ -399,6 +408,7 @@ public final class FeatureFlags {
     public static class DebugFlag extends BooleanFlag {
 
         public final String description;
+        protected boolean mHasBeenChangedAtLeastOnce;
         protected boolean mCurrentValue;
 
         public DebugFlag(String key, boolean defaultValue, String description) {
@@ -416,8 +426,10 @@ public final class FeatureFlags {
         }
 
         public void initialize(Context context) {
-            mCurrentValue = context.getSharedPreferences(FLAGS_PREF_NAME, Context.MODE_PRIVATE)
-                    .getBoolean(key, defaultValue);
+            SharedPreferences prefs =
+                    context.getSharedPreferences(FLAGS_PREF_NAME, Context.MODE_PRIVATE);
+            mHasBeenChangedAtLeastOnce = prefs.contains(key);
+            mCurrentValue = prefs.getBoolean(key, defaultValue);
         }
 
         @Override
diff --git a/src/com/android/launcher3/config/FlagTogglerPrefUi.java b/src/com/android/launcher3/config/FlagTogglerPrefUi.java
index 6729f7434f..2eb6e6df9a 100644
--- a/src/com/android/launcher3/config/FlagTogglerPrefUi.java
+++ b/src/com/android/launcher3/config/FlagTogglerPrefUi.java
@@ -52,12 +52,17 @@ public final class FlagTogglerPrefUi {
         public void putBoolean(String key, boolean value) {
             for (DebugFlag flag : FeatureFlags.getDebugFlags()) {
                 if (flag.key.equals(key)) {
-                    SharedPreferences.Editor editor = mContext.getSharedPreferences(
-                            FLAGS_PREF_NAME, Context.MODE_PRIVATE).edit();
-                    if (value == flag.defaultValue) {
+                    SharedPreferences prefs = mContext.getSharedPreferences(
+                            FLAGS_PREF_NAME, Context.MODE_PRIVATE);
+                    SharedPreferences.Editor editor = prefs.edit();
+                    // We keep the key in the prefs even if it has the default value, because it's a
+                    // signal that it has been changed at one point.
+                    if (!prefs.contains(key) && value == flag.defaultValue) {
                         editor.remove(key).apply();
+                        flag.mHasBeenChangedAtLeastOnce = false;
                     } else {
                         editor.putBoolean(key, value).apply();
+                        flag.mHasBeenChangedAtLeastOnce = true;
                     }
                     updateMenu();
                 }