diff --git a/McBitFont/Form1.cs b/McBitFont/Form1.cs
index 65afe36..4759651 100644
--- a/McBitFont/Form1.cs
+++ b/McBitFont/Form1.cs
@@ -574,9 +574,9 @@ namespace McBitFont {
             }
 
             // Check for Shift / Ctrl keys for straight lines
-            if (ModifierKeys.HasFlag(Keys.Shift)) {
+            if (ModifierKeys.HasFlag(Keys.Shift) && mouseDown) {
                 j = lastY;
-            } else if (ModifierKeys.HasFlag(Keys.Control)) {
+            } else if (ModifierKeys.HasFlag(Keys.Control) && mouseDown) {
                 i = lastX;
             }
             lastX = i;