gdb: add extended-remote support, implement vAttach

sysmodules/rosalina/include/gdb.h

@@ -62,10 +62,11 @@ enum
     GDB_FLAG_SELECTED = 1,
     GDB_FLAG_USED  = 2,
     GDB_FLAG_ALLOCATED_MASK = GDB_FLAG_SELECTED | GDB_FLAG_USED,
-    GDB_FLAG_NOACK = 4,
+    GDB_FLAG_EXTENDED_REMOTE = 4,
-    GDB_FLAG_PROCESS_CONTINUING = 8,
+    GDB_FLAG_NOACK = 8,
-    GDB_FLAG_TERMINATE_PROCESS = 16,
+    GDB_FLAG_PROCESS_CONTINUING = 16,
-    GDB_FLAG_ATTACHED_AT_START = 32,
+    GDB_FLAG_TERMINATE_PROCESS = 32,
+    GDB_FLAG_ATTACHED_AT_START = 64,
 };
 
 typedef enum GDBState
@@ -139,3 +140,4 @@ Result GDB_AttachToProcess(GDBContext *ctx);
 void GDB_DetachFromProcess(GDBContext *ctx);
 
 GDB_DECLARE_HANDLER(Unsupported);
+GDB_DECLARE_HANDLER(EnableExtendedMode);

sysmodules/rosalina/include/gdb/debug.h

@@ -28,6 +28,7 @@
 
 #include "gdb.h"
 
+GDB_DECLARE_VERBOSE_HANDLER(Attach);
 GDB_DECLARE_HANDLER(Detach);
 GDB_DECLARE_HANDLER(Kill);
 GDB_DECLARE_HANDLER(Break);

sysmodules/rosalina/source/gdb.c

@@ -26,6 +26,7 @@
 
 #include "gdb.h"
 #include "gdb/net.h"
+#include "gdb/server.h"
 
 #include "gdb/debug.h"
 
@@ -138,8 +139,6 @@ void GDB_DetachFromProcess(GDBContext *ctx)
     svcKernelSetState(0x10002, ctx->pid, false);
     memset(ctx->svcMask, 0, 32);
 
-    memset(ctx->memoryOsInfoXmlData, 0, sizeof(ctx->memoryOsInfoXmlData));
-    memset(ctx->processesOsInfoXmlData, 0, sizeof(ctx->processesOsInfoXmlData));
     memset(ctx->threadListData, 0, sizeof(ctx->threadListData));
     ctx->threadListDataPos = 0;
 
@@ -184,3 +183,10 @@ GDB_DECLARE_HANDLER(Unsupported)
 {
     return GDB_ReplyEmpty(ctx);
 }
+
+GDB_DECLARE_HANDLER(EnableExtendedMode)
+{
+
+    ctx->flags |= GDB_FLAG_EXTENDED_REMOTE;
+    return GDB_ReplyOk(ctx);
+}

sysmodules/rosalina/source/gdb/debug.c

@@ -25,6 +25,7 @@
 */
 
 #include "gdb/debug.h"
+#include "gdb/server.h"
 #include "gdb/verbose.h"
 #include "gdb/net.h"
 #include "gdb/thread.h"
@@ -35,6 +36,23 @@
 #include <stdlib.h>
 #include <signal.h>
 
+GDB_DECLARE_VERBOSE_HANDLER(Attach)
+{
+    // Extended remote only
+    if (!(ctx->flags & GDB_FLAG_EXTENDED_REMOTE))
+        return GDB_ReplyErrno(ctx, EPERM);
+
+    u32 pid;
+    if(GDB_ParseHexIntegerList(&pid, ctx->commandData, 1, 0) == NULL)
+        return GDB_ReplyErrno(ctx, EILSEQ);
+
+    RecursiveLock_Lock(&ctx->lock);
+    ctx->pid = pid;
+    Result r = GDB_AttachToProcess(ctx);
+    RecursiveLock_Unlock(&ctx->lock);
+    return R_SUCCEEDED(r) ? GDB_SendStopReply(ctx, &ctx->latestDebugEvent) : GDB_ReplyErrno(ctx, EPERM);
+}
+
 /*
     Since we can't select particular threads to continue (and that's uncompliant behavior):
         - if we continue the current thread, continue all threads
@@ -44,6 +62,15 @@
 GDB_DECLARE_HANDLER(Detach)
 {
     ctx->state = GDB_STATE_DETACHING;
+    if (ctx->flags & GDB_FLAG_EXTENDED_REMOTE)
+    {
+        // detach immediately
+        RecursiveLock_Lock(&ctx->lock);
+        svcSignalEvent(ctx->parent->statusUpdated); // note: monitor will be waiting for lock
+        GDB_DetachFromProcess(ctx);
+        ctx->flags = GDB_FLAG_USED;
+        RecursiveLock_Unlock(&ctx->lock);
+    }
     return GDB_ReplyOk(ctx);
 }
 
@@ -51,6 +78,14 @@ GDB_DECLARE_HANDLER(Kill)
 {
     ctx->state = GDB_STATE_DETACHING;
     ctx->flags |= GDB_FLAG_TERMINATE_PROCESS;
+    if (ctx->flags & GDB_FLAG_EXTENDED_REMOTE)
+    {
+        // detach & kill immediately
+        RecursiveLock_Lock(&ctx->lock);
+        svcSignalEvent(ctx->parent->statusUpdated); // note: monitor will be waiting for lock
+        GDB_DetachFromProcess(ctx);
+        RecursiveLock_Unlock(&ctx->lock);
+    }
     return 0;
 }
 
@@ -153,7 +188,7 @@ GDB_DECLARE_VERBOSE_HANDLER(Continue)
 
 GDB_DECLARE_HANDLER(GetStopReason)
 {
-    return GDB_SendStopReply(ctx, &ctx->latestDebugEvent);
+    return ctx->debug != 0 ? GDB_SendStopReply(ctx, &ctx->latestDebugEvent) : GDB_SendPacket(ctx, "W00", 3); // "process exited" if nothing
 }
 
 static int GDB_ParseCommonThreadInfo(char *out, GDBContext *ctx, int sig)

sysmodules/rosalina/source/gdb/server.c

@@ -167,10 +167,11 @@ GDBContext *GDB_FindAllocatedContextByPid(GDBServer *server, u32 pid)
 
 int GDB_AcceptClient(GDBContext *ctx)
 {
-    Result r;
+    Result r = 0;
 
     RecursiveLock_Lock(&ctx->lock);
-    r = GDB_AttachToProcess(ctx);
+    if (ctx->flags & GDB_FLAG_SELECTED)
+        r = GDB_AttachToProcess(ctx);
     RecursiveLock_Unlock(&ctx->lock);
 
     return R_SUCCEEDED(r) ? 0 : -1;
@@ -182,7 +183,6 @@ int GDB_CloseClient(GDBContext *ctx)
     svcSignalEvent(ctx->parent->statusUpdated); // note: monitor will be waiting for lock
     GDB_DetachFromProcess(ctx);
     RecursiveLock_Unlock(&ctx->lock);
-
     return 0;
 }
 
@@ -192,7 +192,7 @@ GDBContext *GDB_GetClient(GDBServer *server, u16 port)
     GDBContext *ctx = NULL;
     for (u32 i = 0; i < MAX_DEBUG; i++)
     {
-        if ((server->ctxs[i].flags & GDB_FLAG_SELECTED) && server->ctxs[i].localPort == port)
+        if (server->ctxs[i].localPort == port)
         {
             ctx = &server->ctxs[i];
             break;
@@ -201,6 +201,31 @@ GDBContext *GDB_GetClient(GDBServer *server, u16 port)
 
     if (ctx != NULL)
     {
+        // Context already tied to a port/selected
+        // Extended remote support disabled
+        if (ctx->flags & GDB_FLAG_USED)
+        {
+            GDB_UnlockAllContexts(server);
+            return NULL;
+        }
+        ctx->flags |= GDB_FLAG_USED;
+        ctx->state = GDB_STATE_CONNECTED;
+        ctx->parent = server;
+    }
+    else
+    {
+        // Grab a free context
+        u32 id;
+        for(id = 0; id < MAX_DEBUG && (server->ctxs[id].flags & GDB_FLAG_ALLOCATED_MASK); id++);
+        if(id < MAX_DEBUG)
+            ctx = &server->ctxs[id];
+        else
+        {
+            GDB_UnlockAllContexts(server);
+            return NULL;
+        }
+
+        ctx->localPort = port;
         ctx->flags |= GDB_FLAG_USED;
         ctx->state = GDB_STATE_CONNECTED;
         ctx->parent = server;
@@ -221,6 +246,10 @@ void GDB_ReleaseClient(GDBServer *server, GDBContext *ctx)
 
     ctx->catchThreadEvents = false;
 
+    memset(&ctx->latestDebugEvent, 0, sizeof(DebugEventInfo));
+    memset(ctx->memoryOsInfoXmlData, 0, sizeof(ctx->memoryOsInfoXmlData));
+    memset(ctx->processesOsInfoXmlData, 0, sizeof(ctx->processesOsInfoXmlData));
+
     RecursiveLock_Unlock(&ctx->lock);
 }
 
@@ -231,6 +260,7 @@ static const struct
 } gdbCommandHandlers[] =
 {
     { '?', GDB_HANDLER(GetStopReason) },
+    { '!', GDB_HANDLER(EnableExtendedMode) },
     { 'c', GDB_HANDLER(Continue) },
     { 'C', GDB_HANDLER(Continue) },
     { 'D', GDB_HANDLER(Detach) },
@@ -292,7 +322,7 @@ int GDB_DoPacket(GDBContext *ctx)
 
     RecursiveLock_Unlock(&ctx->lock);
     if(ctx->state == GDB_STATE_DETACHING)
-        return -1;
+        return (ctx->flags & GDB_FLAG_EXTENDED_REMOTE) ? ret : -1;
 
     if((oldFlags & GDB_FLAG_PROCESS_CONTINUING) && !(ctx->flags & GDB_FLAG_PROCESS_CONTINUING))
     {

sysmodules/rosalina/source/gdb/verbose.c

@@ -34,6 +34,7 @@ static const struct
     GDBCommandHandler handler;
 } gdbVerboseCommandHandlers[] =
 {
+    { "Attach", GDB_VERBOSE_HANDLER(Attach) },
     { "Cont?", GDB_VERBOSE_HANDLER(ContinueSupported) },
     { "Cont",  GDB_VERBOSE_HANDLER(Continue) },
     { "MustReplyEmpty", GDB_HANDLER(Unsupported) },

sysmodules/rosalina/source/gdb/xfer.c

@@ -144,17 +144,17 @@ GDB_DECLARE_XFER_OSDATA_HANDLER(Processes)
         if(ctx->processesOsInfoXmlData[0] == 0)
         {
             static const char header[] =
-            "<?xml version=\"1.0\"?>\n"
+            /*"<?xml version=\"1.0\"?>"
-            "<!DOCTYPE target SYSTEM \"osdata.dtd\">\n"
+            "<!DOCTYPE target SYSTEM \"osdata.dtd\">" IDA rejects the xml header*/
-            "<osdata type=\"processes\">\n";
+            "<osdata type=\"processes\">";
 
             static const char item[] =
-            "  <item>\n"
+            "<item>"
-            "    <column name=\"pid\">%lu</column>\n"
+            "<column name=\"pid\">%lu</column>"
-            "    <column name=\"command\">%s</column>\n"
+            "<column name=\"command\">%s</column>"
-            "  </item>\n";
+            "</item>";
 
-            static const char footer[] = "</osdata>\n";
+            static const char footer[] = "</osdata>";
 
             int n;
             u32 pos = 0;