diff --git a/config/optimize b/config/optimize
index 9c57fcaa92..e3f8b475f4 100644
--- a/config/optimize
+++ b/config/optimize
@@ -1,16 +1,6 @@
 # Linker hash-style is set to gnu via gcc default
 LD_OPTIM="-Wl,--as-needed"
 
-if [ "${BUILD_WITH_DEBUG}" = "yes" ]; then
-  TARGET_CFLAGS="$TARGET_CFLAGS -ggdb"
-  TARGET_CXXFLAGS="$TARGET_CXXFLAGS -ggdb"
-  TARGET_LDFLAGS="$TARGET_LDFLAGS -ggdb"
-else
-  TARGET_CFLAGS="$TARGET_CFLAGS -fomit-frame-pointer"
-  TARGET_CXXFLAGS="$TARGET_CXXFLAGS -fomit-frame-pointer"
-  TARGET_LDFLAGS="$TARGET_LDFLAGS"
-fi
-
 NINJA_OPTS=""
 
 TARGET_CPPFLAGS=""
diff --git a/config/options b/config/options
index 63239b6151..f05b8f5802 100644
--- a/config/options
+++ b/config/options
@@ -112,3 +112,13 @@ fi
 
 # set package metadata
 source_package "${1}"
+
+if [ "${BUILD_WITH_DEBUG}" = "yes" ]; then
+  TARGET_CFLAGS="$TARGET_CFLAGS -ggdb"
+  TARGET_CXXFLAGS="$TARGET_CXXFLAGS -ggdb"
+  TARGET_LDFLAGS="$TARGET_LDFLAGS -ggdb"
+else
+  TARGET_CFLAGS="$TARGET_CFLAGS -fomit-frame-pointer"
+  TARGET_CXXFLAGS="$TARGET_CXXFLAGS -fomit-frame-pointer"
+  TARGET_LDFLAGS="$TARGET_LDFLAGS"
+fi