diff --git a/configure.in b/configure.in
index 1657b1a65..9cb9cf520 100644
--- a/configure.in
+++ b/configure.in
@@ -212,7 +212,6 @@ AC_ARG_ENABLE(compiler-flags, [  --disable-compiler-flags
 
 if test "$enable_compiler_flags" != "no"; then
   AC_C_FLAG([-Wall])
-  AC_C_FLAG([-O2])
   AC_C_FLAG([-Wextra],[AC_C_FLAG([-W])])
   AC_C_FLAG([-Wno-unused-parameter])
   AC_C_FLAG([-std=gnu99])
@@ -220,12 +219,16 @@ if test "$enable_compiler_flags" != "no"; then
   AC_CXX_FLAG([-Wextra],[AC_CXX_FLAG([-W])])
   AC_CXX_FLAG([-Wno-unused-parameter])
   AC_CXX_FLAG([-Wno-long-long])
-  AC_CXX_FLAG([-O2])
   AC_CXX_FLAG([-fpermissive])
   AC_CXX_FLAG([-fno-strict-aliasing])
   AC_CXX_FLAG([-std=c++98])
 fi
 
+if test "$enable_compiler_flags" != "no" || test "$enable_debug" != "yes"; then
+  AC_C_FLAG([-O2])
+  AC_CXX_FLAG([-O2])
+fi
+
 ######################################
 # Check Headers / Features / Libraries
 ######################################