diff --git a/aegisub/src/dialog_colorpicker.cpp b/aegisub/src/dialog_colorpicker.cpp
index 794632926..10f2a9899 100644
--- a/aegisub/src/dialog_colorpicker.cpp
+++ b/aegisub/src/dialog_colorpicker.cpp
@@ -470,8 +470,11 @@ wxString ColorPickerRecent::StoreToString()
 
 void ColorPickerRecent::AddColor(wxColour color)
 {
-	colors.erase(remove(colors.begin(), colors.end(), color), colors.end());
-	colors.insert(colors.begin(), color);
+	std::vector<wxColour>::iterator existing = find(colors.begin(), colors.end(), color);
+	if (existing != colors.end())
+		rotate(colors.begin(), existing, existing + 1);
+	else
+		colors.insert(colors.begin(), color);
 
 	background_valid = false;