winedbg: Escape special characters in GDB packet reply.

There are four special characters in GDB's remote serial protocol:

- '$' (0x24): start of packet
- '}' (0x7D): escape
- '*' (0x2A): run-length encoding repeat count delimiter
- '#' (0x23): end of packet; start of checksum

In particular, the '#' and '}' characters are problematic since they
are often used in library filenames.  A few examples:

- %SystemRoot%\assembly\NativeImages_v[.NET ver]\[module+hash]#\*\*.dll
- {CLSID or UUID}\*\.dll

To make GDB happy with those filenames, we scan for those characters and
escape them properly.

While we are at it, also remove the assert in the packet_reply function
that checks for '$' and '#' in the packet payload.

Signed-off-by: Jinoh Kang <jinoh.kang.kr@gmail.com>
Signed-off-by: Rémi Bernon <rbernon@codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard@winehq.org>

programs/winedbg/gdbproxy.c

@@ -739,18 +739,55 @@ static void packet_reply_val(struct gdb_context* gdbctx, ULONG_PTR val, int len)
     }
 }
 
-static inline void packet_reply_add(struct gdb_context* gdbctx, const char* str)
+static const unsigned char gdb_special_chars_lookup_table[4] = {
+    /* The characters should be indexed by its value modulo table length. */
+
+    0x24, /* $: 001001|00 */
+    0x7D, /* }: 011111|01 */
+    0x2A, /* *: 001010|10 */
+    0x23  /* #: 001000|11 */
+};
+
+static inline BOOL is_gdb_special_char(unsigned char val)
 {
-    int len = strlen(str);
-    packet_reply_grow(gdbctx, len);
-    memcpy(&gdbctx->out_buf[gdbctx->out_len], str, len);
-    gdbctx->out_len += len;
+    /* A note on the GDB special character scanning code:
+     *
+     * We cannot use strcspn() since we plan to transmit binary data in
+     * packet reply, which can contain NULL (0x00) bytes.  We also don't want
+     * to slow down memory dump transfers.  Therefore, we use a tiny lookup
+     * table that contains all the four special characters to speed up scanning.
+     */
+    const size_t length = ARRAY_SIZE(gdb_special_chars_lookup_table);
+    return gdb_special_chars_lookup_table[val % length] == val;
+}
+
+static void packet_reply_add(struct gdb_context* gdbctx, const char* str)
+{
+    const unsigned char *ptr = (unsigned char *)str, *curr;
+
+    while (*ptr)
+    {
+        curr = ptr;
+
+        while (*ptr && !is_gdb_special_char(*ptr))
+            ptr++;
+
+        packet_reply_grow(gdbctx, ptr - curr);
+        memcpy(&gdbctx->out_buf[gdbctx->out_len], curr, ptr - curr);
+        gdbctx->out_len += ptr - curr;
+        if (!*ptr) break;
+
+        packet_reply_grow(gdbctx, 2);
+        gdbctx->out_buf[gdbctx->out_len++] = 0x7D;
+        gdbctx->out_buf[gdbctx->out_len++] = 0x20 ^ *ptr++;
+    }
 }
 
 static void packet_reply_open(struct gdb_context* gdbctx)
 {
     assert(gdbctx->out_curr_packet == -1);
-    packet_reply_add(gdbctx, "$");
+    packet_reply_grow(gdbctx, 1);
+    gdbctx->out_buf[gdbctx->out_len++] = '$';
     gdbctx->out_curr_packet = gdbctx->out_len;
 }
 
@@ -760,7 +797,8 @@ static void packet_reply_close(struct gdb_context* gdbctx)
     int plen;
 
     plen = gdbctx->out_len - gdbctx->out_curr_packet;
-    packet_reply_add(gdbctx, "#");
+    packet_reply_grow(gdbctx, 1);
+    gdbctx->out_buf[gdbctx->out_len++] = '#';
     cksum = checksum(&gdbctx->out_buf[gdbctx->out_curr_packet], plen);
     packet_reply_hex_to(gdbctx, &cksum, 1);
     gdbctx->out_curr_packet = -1;
@@ -799,8 +837,6 @@ static enum packet_return packet_reply(struct gdb_context* gdbctx, const char* p
 {
     packet_reply_open(gdbctx);
 
-    assert(strchr(packet, '$') == NULL && strchr(packet, '#') == NULL);
-
     packet_reply_add(gdbctx, packet);
 
     packet_reply_close(gdbctx);