Implement HBLDR_DebugNextApplicationByForce

also prevent port 4003 (next application) from being extended-remote

sysmodules/rosalina/source/gdb.c

@@ -211,7 +211,11 @@ GDB_DECLARE_HANDLER(Unsupported)
 
 GDB_DECLARE_HANDLER(EnableExtendedMode)
 {
-
+    if (ctx->localPort >= GDB_PORT_BASE && ctx->localPort < GDB_PORT_BASE + MAX_DEBUG)
-    ctx->flags |= GDB_FLAG_EXTENDED_REMOTE;
+    {
-    return GDB_ReplyOk(ctx);
+        ctx->flags |= GDB_FLAG_EXTENDED_REMOTE;
+        return GDB_ReplyOk(ctx);
+    }
+    else
+        return GDB_ReplyEmpty(ctx);
 }

sysmodules/rosalina/source/gdb/server.c

@@ -229,7 +229,7 @@ GDBContext *GDB_GetClient(GDBServer *server, u16 port)
         ctx->state = GDB_STATE_CONNECTED;
         ctx->parent = server;
     }
-    else
+    else if (port >= GDB_PORT_BASE && port < GDB_PORT_BASE + MAX_DEBUG)
     {
         // Grab a free context
         u32 id;

sysmodules/rosalina/source/hbloader.c

@@ -24,7 +24,7 @@
 *         reasonable ways as different from the original version.
 */
 
-/* This file was entirely written by fincs */
+/* This file was mostly written by fincs */
 
 #include <3ds.h>
 #include "hbloader.h"
@@ -33,8 +33,14 @@
 #include "csvc.h"
 #include "memory.h"
 
+#include "gdb/server.h"
+#include "pmdbgext.h"
+
 #define MAP_BASE 0x10000000
 
+extern GDBContext *nextApplicationGdbCtx;
+extern GDBServer gdbServer;
+
 static const char serviceList[32][8] =
 {
     "APT:U",
@@ -144,7 +150,7 @@ static u16 *u16_strncpy(u16 *dest, const u16 *src, u32 size)
 void HBLDR_HandleCommands(void *ctx)
 {
     (void)ctx;
-    Result res;
+    Result res = 0;
     IFile file;
     u32 *cmdbuf = getThreadCommandBuffer();
     switch (cmdbuf[0] >> 16)
@@ -309,6 +315,37 @@ void HBLDR_HandleCommands(void *ctx)
             cmdbuf[3] = (u32)exhi;
             break;
         }
+        case 5: // DebugNextApplicationByForce
+        {
+            res = 0;
+            GDB_LockAllContexts(&gdbServer);
+            if (nextApplicationGdbCtx != NULL)
+                res = MAKERESULT(RL_PERMANENT, RS_NOP, RM_LDR, RD_ALREADY_DONE);
+            else if (gdbServer.referenceCount == 0)
+                res = MAKERESULT(RL_PERMANENT, RS_INVALIDSTATE, RM_LDR, RD_NOT_INITIALIZED);
+            else
+            {
+                nextApplicationGdbCtx = GDB_SelectAvailableContext(&gdbServer, GDB_PORT_BASE + 3, GDB_PORT_BASE + 4);
+                if (nextApplicationGdbCtx != NULL)
+                {
+                    nextApplicationGdbCtx->debug = 0;
+                    nextApplicationGdbCtx->pid = 0xFFFFFFFF;
+                    res = PMDBG_DebugNextApplicationByForce();
+                    if (R_FAILED(res))
+                    {
+                        nextApplicationGdbCtx->flags = 0;
+                        nextApplicationGdbCtx->localPort = 0;
+                        nextApplicationGdbCtx = NULL;
+                    }
+                }
+                else
+                    res = MAKERESULT(RL_PERMANENT, RS_OUTOFRESOURCE, RM_LDR, RD_OUT_OF_RANGE);
+            }
+            GDB_UnlockAllContexts(&gdbServer);
+            cmdbuf[1] = res;
+            cmdbuf[0] = IPC_MakeHeader(5, 1, 0);
+            break;
+        }
         default:
         {
             error(cmdbuf, 0xD900182F);

sysmodules/rosalina/source/menus/debugger.c

@@ -52,7 +52,7 @@ static u8 ALIGN(8) debuggerDebugThreadStack[0x2000];
 
 GDBServer gdbServer = { 0 };
 
-static GDBContext *nextApplicationGdbCtx = NULL;
+GDBContext *nextApplicationGdbCtx = NULL;
 
 void debuggerSocketThreadMain(void);
 MyThread *debuggerCreateSocketThread(void)
@@ -170,29 +170,34 @@ void DebuggerMenu_DebugNextApplicationByForce(void)
     Result res = 0;
     char buf[256];
 
-    if(initialized)
+    GDB_LockAllContexts(&gdbServer);
+
+    if (nextApplicationGdbCtx != NULL)
+        strcpy(buf, "Operation already performed.");
+    else if(initialized)
     {
-        res = PMDBG_DebugNextApplicationByForce();
+        nextApplicationGdbCtx = GDB_SelectAvailableContext(&gdbServer, GDB_PORT_BASE + 3, GDB_PORT_BASE + 4);
-        if(R_SUCCEEDED(res))
+        if (nextApplicationGdbCtx != NULL)
         {
-            GDB_LockAllContexts(&gdbServer);
+            nextApplicationGdbCtx->debug = 0;
-            if (nextApplicationGdbCtx == NULL)
+            nextApplicationGdbCtx->pid = 0xFFFFFFFF;
-                nextApplicationGdbCtx = GDB_SelectAvailableContext(&gdbServer, GDB_PORT_BASE + 3, GDB_PORT_BASE + 4);
+            res = PMDBG_DebugNextApplicationByForce();
-            if (nextApplicationGdbCtx != NULL)
+            if(R_SUCCEEDED(res))
-            {
-                nextApplicationGdbCtx->debug = 0;
-                nextApplicationGdbCtx->pid = 0xFFFFFFFF;
                 sprintf(buf, "Operation succeeded.\nUse port %d to connect to the next launched\napplication.", nextApplicationGdbCtx->localPort);
-            }
             else
-                strcpy(buf, "Failed to allocate a slot.\nPlease unselect a process in the process list first");
+            {
-            GDB_UnlockAllContexts(&gdbServer);
+                nextApplicationGdbCtx->flags = 0;
+                nextApplicationGdbCtx->localPort = 0;
+                nextApplicationGdbCtx = NULL;
+                    sprintf(buf, "Operation failed (0x%08lx).", (u32)res);
+            }
         }
         else
-            sprintf(buf, "Operation failed (0x%08lx).", (u32)res);
+            strcpy(buf, "Failed to allocate a slot.\nPlease unselect a process in the process list first");
     }
     else
         strcpy(buf, "Debugger not enabled.");
+    GDB_UnlockAllContexts(&gdbServer);
 
     do
     {