Add support to force-debug applications before they start running code

sysmodules/rosalina/include/gdb.h

@@ -63,6 +63,7 @@ typedef enum GDBFlags
     GDB_FLAG_USED  = 2,
     GDB_FLAG_PROCESS_CONTINUING = 4,
     GDB_FLAG_TERMINATE_PROCESS = 8,
+    GDB_FLAG_ATTACHED_AT_START = 16,
 } GDBFlags;
 
 typedef enum GDBState
@@ -94,8 +95,8 @@ typedef struct GDBContext
     Handle debug;
     ThreadInfo threadInfos[MAX_DEBUG_THREAD];
     u32 nbThreads;
-
     u32 currentThreadId, selectedThreadId, selectedThreadIdForContinuing;
+    u32 totalNbCreatedThreads;
 
     Handle clientAcceptedEvent, continuedEvent;
     Handle eventToWaitFor;

sysmodules/rosalina/include/menus/debugger.h

@@ -31,5 +31,7 @@
 
 extern Menu debuggerMenu;
 
+void debuggerSetNextApplicationDebugHandle(Handle debug);
 void DebuggerMenu_EnableDebugger(void);
 void DebuggerMenu_DisableDebugger(void);
+void DebuggerMenu_DebugNextApplicationByForce(void);

sysmodules/rosalina/include/sock_util.h

@@ -31,7 +31,7 @@
 #include <poll.h>
 #include <netinet/in.h>
 
-#define MAX_PORTS 3
+#define MAX_PORTS (3+1)
 #define MAX_CTXS  (2 * MAX_PORTS)
 
 struct sock_server;

sysmodules/rosalina/source/gdb/debug.c

@@ -196,12 +196,19 @@ void GDB_PreprocessDebugEvent(GDBContext *ctx, DebugEventInfo *info)
 {
     switch(info->type)
     {
+        case DBGEVENT_ATTACH_PROCESS:
+        {
+            ctx->pid = info->attach_process.process_id;
+            break;
+        }
+
         case DBGEVENT_ATTACH_THREAD:
         {
             if(ctx->nbThreads == MAX_DEBUG_THREAD)
                 svcBreak(USERBREAK_ASSERT);
             else
             {
+                ++ctx->totalNbCreatedThreads;
                 ctx->threadInfos[ctx->nbThreads].id = info->thread_id;
                 ctx->threadInfos[ctx->nbThreads++].tls = info->attach_thread.thread_local_storage;
             }
@@ -286,7 +293,14 @@ int GDB_SendStopReply(GDBContext *ctx, const DebugEventInfo *info)
 
         case DBGEVENT_ATTACH_THREAD:
         {
-            if(info->attach_thread.creator_thread_id == 0 || !ctx->catchThreadEvents)
+            if((ctx->flags & GDB_FLAG_ATTACHED_AT_START) && ctx->totalNbCreatedThreads == 1)
+            {
+                // Main thread created
+                ctx->currentThreadId = info->thread_id;
+                GDB_ParseCommonThreadInfo(buffer, ctx, SIGINT);
+                return GDB_SendFormattedPacket(ctx, "%s", buffer);
+            }
+            else if(info->attach_thread.creator_thread_id == 0 || !ctx->catchThreadEvents)
                 break; // Dismissed
             else
             {
@@ -483,7 +497,6 @@ int GDB_HandleDebugEvents(GDBContext *ctx)
                                 (info.type == DBGEVENT_EXIT_THREAD && (info.exit_thread.reason >= EXITTHREAD_EVENT_EXIT_PROCESS || !ctx->catchThreadEvents)) ||
                                 info.type == DBGEVENT_EXIT_PROCESS || !(info.flags & 1);
 
-
     if(continueAutomatically)
     {
         Result r = 0;

sysmodules/rosalina/source/gdb/server.c

@@ -87,6 +87,9 @@ void GDB_RunServer(GDBServer *server)
     server_bind(&server->super, GDB_PORT_BASE);
     server_bind(&server->super, GDB_PORT_BASE + 1);
     server_bind(&server->super, GDB_PORT_BASE + 2);
+
+    server_bind(&server->super, GDB_PORT_BASE + 3); // next application
+
     server_run(&server->super);
 }
 
@@ -152,14 +155,28 @@ GDBContext *GDB_SelectAvailableContext(GDBServer *server, u16 minPort, u16 maxPo
 int GDB_AcceptClient(GDBContext *ctx)
 {
     RecursiveLock_Lock(&ctx->lock);
-    Result r = svcDebugActiveProcess(&ctx->debug, ctx->pid);
+    Result r;
+
+    // Two cases: attached during execution, or started attached
+    // The second case will have, after RunQueuedProcess: attach process, debugger break, attach thread (with creator = 0)
+
+    if (!(ctx->flags & GDB_FLAG_ATTACHED_AT_START))
+        svcDebugActiveProcess(&ctx->debug, ctx->pid);
+    else
+    {
+        r = 0;
+    }
     if(R_SUCCEEDED(r))
     {
+        // Note: ctx->pid will be (re)set while processing 'attach process'
         DebugEventInfo *info = &ctx->latestDebugEvent;
         ctx->state = GDB_STATE_CONNECTED;
         ctx->processExited = ctx->processEnded = false;
         ctx->latestSentPacketSize = 0;
-        while(R_SUCCEEDED(svcGetProcessDebugEvent(info, ctx->debug)) && info->type != DBGEVENT_EXCEPTION &&
+        if (!(ctx->flags & GDB_FLAG_ATTACHED_AT_START))
+        {
+            while(R_SUCCEEDED(svcGetProcessDebugEvent(info, ctx->debug)) &&
+                info->type != DBGEVENT_EXCEPTION &&
                 info->exception.type != EXCEVENT_ATTACH_BREAK)
             {
                 GDB_PreprocessDebugEvent(ctx, info);
@@ -167,6 +184,25 @@ int GDB_AcceptClient(GDBContext *ctx)
             }
         }
         else
+        {
+            // Attach process, debugger break
+            for(u32 i = 0; i < 2; i++)
+            {
+                if (R_FAILED(svcGetProcessDebugEvent(info, ctx->debug)))
+                    return -1;
+                GDB_PreprocessDebugEvent(ctx, info);
+                if (R_FAILED(svcContinueDebugEvent(ctx->debug, ctx->continueFlags)))
+                    return -1;
+            }
+
+            svcWaitSynchronization(ctx->debug, -1LL);
+            if (R_FAILED(svcGetProcessDebugEvent(info, ctx->debug)))
+                return -1; //svcBreak(0);
+            // Attach thread
+            GDB_PreprocessDebugEvent(ctx, info);
+        }
+    }
+    else
     {
         RecursiveLock_Unlock(&ctx->lock);
         return -1;
@@ -217,8 +253,6 @@ GDBContext *GDB_GetClient(GDBServer *server, u16 port)
 {
     GDB_LockAllContexts(server);
     GDBContext *ctx = NULL;
-    if (port >= GDB_PORT_BASE && port < GDB_PORT_BASE + MAX_DEBUG)
-    {
     for (u32 i = 0; i < MAX_DEBUG; i++)
     {
         if ((server->ctxs[i].flags & GDB_FLAG_SELECTED) && server->ctxs[i].localPort == port)
@@ -227,7 +261,6 @@ GDBContext *GDB_GetClient(GDBServer *server, u16 port)
             break;
         }
     }
-    }
 
     if (ctx != NULL)
     {
@@ -277,6 +310,7 @@ void GDB_ReleaseClient(GDBServer *server, GDBContext *ctx)
     ctx->pid = 0;
     ctx->currentThreadId = ctx->selectedThreadId = ctx->selectedThreadIdForContinuing = 0;
     ctx->nbThreads = 0;
+    ctx->totalNbCreatedThreads = 0;
     memset(ctx->threadInfos, 0, sizeof(ctx->threadInfos));
     ctx->catchThreadEvents = false;
     ctx->enableExternalMemoryAccess = false;

sysmodules/rosalina/source/main.c

@@ -35,6 +35,7 @@
 #include "MyThread.h"
 #include "menus/process_patches.h"
 #include "menus/miscellaneous.h"
+#include "menus/debugger.h"
 #include "menus/screen_filters.h"
 
 static Result stealFsReg(void)
@@ -154,6 +155,14 @@ static void handleTermNotification(u32 notificationId)
     svcSignalEvent(terminationRequestEvent);
 }
 
+static void handleNextApplicationDebuggedByForce(u32 notificationId)
+{
+    (void)notificationId;
+    Handle debug = 0;
+    PMDBG_RunQueuedProcess(&debug);
+    debuggerSetNextApplicationDebugHandle(debug);
+}
+
 static const ServiceManagerServiceEntry services[] = {
     { "err:f",  1, ERRF_HandleCommands,  true },
     { "hb:ldr", 2, HBLDR_HandleCommands, true },
@@ -161,7 +170,8 @@ static const ServiceManagerServiceEntry services[] = {
 };
 
 static const ServiceManagerNotificationEntry notifications[] = {
-    { 0x100, handleTermNotification },
+    { 0x100 , handleTermNotification                },
+    { 0x1000, handleNextApplicationDebuggedByForce  },
     { 0x000, NULL },
 };
 

sysmodules/rosalina/source/menus/debugger.c

@@ -29,6 +29,7 @@
 #include "draw.h"
 #include "minisoc.h"
 #include "fmt.h"
+#include "pmdbgext.h"
 #include "gdb/server.h"
 #include "gdb/debug.h"
 #include "gdb/monitor.h"
@@ -36,10 +37,11 @@
 
 Menu debuggerMenu = {
     "Debugger options menu",
-    .nbItems = 2,
+    .nbItems = 3,
     {
         { "Enable debugger",                        METHOD, .method = &DebuggerMenu_EnableDebugger  },
-        { "Disable debugger", METHOD, .method = &DebuggerMenu_DisableDebugger }
+        { "Disable debugger",                       METHOD, .method = &DebuggerMenu_DisableDebugger },
+        { "Force-debug next application at launch", METHOD, .method = &DebuggerMenu_DebugNextApplicationByForce },
     }
 };
 
@@ -50,6 +52,8 @@ static u8 ALIGN(8) debuggerDebugThreadStack[0x2000];
 
 GDBServer gdbServer = { 0 };
 
+static GDBContext *nextApplicationGdbCtx = NULL;
+
 void debuggerSocketThreadMain(void);
 MyThread *debuggerCreateSocketThread(void)
 {
@@ -64,6 +68,18 @@ MyThread *debuggerCreateDebugThread(void)
     return &debuggerDebugThread;
 }
 
+void debuggerSetNextApplicationDebugHandle(Handle debug)
+{
+    GDB_LockAllContexts(&gdbServer);
+    nextApplicationGdbCtx->debug = debug;
+    if (debug == 0)
+        nextApplicationGdbCtx->flags = (GDBFlags)0;
+    else
+        nextApplicationGdbCtx->flags |= GDB_FLAG_ATTACHED_AT_START;
+    nextApplicationGdbCtx = NULL;
+    GDB_UnlockAllContexts(&gdbServer);
+}
+
 void DebuggerMenu_EnableDebugger(void)
 {
     bool done = false, alreadyEnabled = gdbServer.super.running;
@@ -148,6 +164,47 @@ void DebuggerMenu_DisableDebugger(void)
     while(!(waitInput() & BUTTON_B) && !terminationRequest);
 }
 
+void DebuggerMenu_DebugNextApplicationByForce(void)
+{
+    bool initialized = gdbServer.referenceCount != 0;
+    Result res = 0;
+    char buf[256];
+
+    if(initialized)
+    {
+        res = PMDBG_DebugNextApplicationByForce();
+        if(R_SUCCEEDED(res))
+        {
+            GDB_LockAllContexts(&gdbServer);
+            if (nextApplicationGdbCtx == NULL)
+                nextApplicationGdbCtx = GDB_SelectAvailableContext(&gdbServer, GDB_PORT_BASE + 3, GDB_PORT_BASE + 4);
+            if (nextApplicationGdbCtx != NULL)
+            {
+                nextApplicationGdbCtx->debug = 0;
+                nextApplicationGdbCtx->pid = 0xFFFFFFFF;
+                sprintf(buf, "Operation succeeded.\nUse port %d to connect to the next launched\napplication.", nextApplicationGdbCtx->localPort);
+            }
+            else
+                strcpy(buf, "Failed to allocate a slot.\nPlease unselect a process in the process list first");
+            GDB_UnlockAllContexts(&gdbServer);
+        }
+        else
+            sprintf(buf, "Operation failed (0x%08lx).", (u32)res);
+    }
+    else
+        strcpy(buf, "Debugger not enabled.");
+
+    do
+    {
+        Draw_Lock();
+        Draw_DrawString(10, 10, COLOR_TITLE, "Debugger options menu");
+        Draw_DrawString(10, 30, COLOR_WHITE, buf);
+        Draw_FlushFramebuffer();
+        Draw_Unlock();
+    }
+    while(!(waitInput() & BUTTON_B) && !terminationRequest);
+}
+
 void debuggerSocketThreadMain(void)
 {
     GDB_IncrementServerReferenceCount(&gdbServer);

sysmodules/rosalina/source/menus/process_list.c

@@ -77,7 +77,7 @@ static inline int ProcessListMenu_FormatInfoLine(char *out, const ProcessInfo *i
             checkbox = "(A) ";
             sprintf(commentBuf, "Remote: %hhu.%hhu.%hhu.%hhu", addr[0], addr[1], addr[2], addr[3]);
         }
-        else
+        else if (gdbServer.ctxs[id].localPort >= GDB_PORT_BASE && gdbServer.ctxs[id].localPort < GDB_PORT_BASE + MAX_DEBUG)
         {
             checkbox = "(W) ";
             sprintf(commentBuf, "Port: %hu", gdbServer.ctxs[id].localPort);
@@ -606,7 +606,7 @@ static inline void ProcessListMenu_HandleSelected(const ProcessInfo *info)
             while(ctx->super.should_close)
                 svcSleepThread(12 * 1000 * 1000LL);
         }
-        else
+        else if (gdbServer.ctxs[id].localPort >= GDB_PORT_BASE && gdbServer.ctxs[id].localPort < GDB_PORT_BASE + MAX_DEBUG)
         {
             RecursiveLock_Lock(&ctx->lock);
             ctx->flags &= ~GDB_FLAG_SELECTED;