rosalina: implement 800px top-screen screenshot, etc. Fixes #1443

sysmodules/rosalina/include/draw.h

@@ -90,7 +90,8 @@ u32 Draw_DrawFormattedString(u32 posX, u32 posY, u32 color, const char *fmt, ...
 
 void Draw_FillFramebuffer(u32 value);
 void Draw_ClearFramebuffer(void);
-u32 Draw_AllocateFramebufferCache(void);
+Result Draw_AllocateFramebufferCache(u32 size);
+Result Draw_AllocateFramebufferCacheForScreenshot(u32 size);
 void Draw_FreeFramebufferCache(void);
 void *Draw_GetFramebufferCache(void);
 u32 Draw_GetFramebufferCacheSize(void);
@@ -98,6 +99,8 @@ u32 Draw_SetupFramebuffer(void);
 void Draw_RestoreFramebuffer(void);
 void Draw_FlushFramebuffer(void);
 u32 Draw_GetCurrentFramebufferAddress(bool top, bool left);
+// Width is actually height as the 3ds screen is rotated 90 degrees
+void Draw_GetCurrentScreenInfo(u32 *width, bool *is3d, bool top);
 
 void Draw_CreateBitmapHeader(u8 *dst, u32 width, u32 heigth);
-void Draw_ConvertFrameBufferLines(u8 *buf, u32 startingLine, u32 numLines, bool top, bool left);
+void Draw_ConvertFrameBufferLines(u8 *buf, u32 width, u32 startingLine, u32 numLines, bool top, bool left);

sysmodules/rosalina/source/draw.c

@@ -129,18 +129,19 @@ void Draw_ClearFramebuffer(void)
     Draw_FillFramebuffer(0);
 }
 
-u32 Draw_AllocateFramebufferCache(void)
+Result Draw_AllocateFramebufferCache(u32 size)
 {
-    // Try to see how much we can allocate...
     // Can't use fbs in FCRAM when Home Menu is active (AXI config related maybe?)
     u32 addr = 0x0D000000;
     u32 tmp;
-    u32 minSize = (FB_BOTTOM_SIZE + 0xFFF) & ~0xFFF;
-    u32 maxSize = (FB_SCREENSHOT_SIZE + 0xFFF) & ~0xFFF;
-    u32 remaining = (u32)osGetMemRegionFree(MEMREGION_SYSTEM);
-    u32 size = remaining < maxSize ? remaining : maxSize;
 
-    if (size < minSize || R_FAILED(svcControlMemoryEx(&tmp, addr, 0, size, MEMOP_ALLOC, MEMREGION_SYSTEM | MEMPERM_READ | MEMPERM_WRITE, true)))
+    size = (size + 0xFFF) >> 12 << 12; // round-up
+
+    if (framebufferCache != NULL)
+        __builtin_trap();
+
+    Result res = svcControlMemoryEx(&tmp, addr, 0, size, MEMOP_ALLOC, MEMREGION_SYSTEM | MEMPERM_READWRITE, true);
+    if (R_FAILED(res))
     {
         framebufferCache = NULL;
         framebufferCacheSize = 0;
@@ -151,12 +152,20 @@ u32 Draw_AllocateFramebufferCache(void)
         framebufferCacheSize = size;
     }
 
-    return framebufferCacheSize;
+    return res;
+}
+
+Result Draw_AllocateFramebufferCacheForScreenshot(u32 size)
+{
+    u32 remaining = (u32)osGetMemRegionFree(MEMREGION_SYSTEM);
+    u32 sz = remaining < size ? remaining : size;
+    return Draw_AllocateFramebufferCache(sz);
 }
 
 void Draw_FreeFramebufferCache(void)
 {
     u32 tmp;
+    if (framebufferCache != NULL)
         svcControlMemory(&tmp, (u32)framebufferCache, 0, framebufferCacheSize, MEMOP_FREE, 0);
     framebufferCacheSize = 0;
     framebufferCache = NULL;
@@ -180,13 +189,19 @@ u32 Draw_SetupFramebuffer(void)
     memcpy(framebufferCache, FB_BOTTOM_VRAM_ADDR, FB_BOTTOM_SIZE);
     Draw_ClearFramebuffer();
     Draw_FlushFramebuffer();
+
+    u32 format = GPU_FB_BOTTOM_FMT;
+
     gpuSavedFramebufferAddr1 = GPU_FB_BOTTOM_ADDR_1;
     gpuSavedFramebufferAddr2 = GPU_FB_BOTTOM_ADDR_2;
-    gpuSavedFramebufferFormat = GPU_FB_BOTTOM_FMT;
+    gpuSavedFramebufferFormat = format;
     gpuSavedFramebufferStride = GPU_FB_BOTTOM_STRIDE;
 
+    format = (format & ~7) | GSP_RGB565_OES;
+    format |= 3 << 8; // set VRAM bits
+
     GPU_FB_BOTTOM_ADDR_1 = GPU_FB_BOTTOM_ADDR_2 = FB_BOTTOM_VRAM_PA;
-    GPU_FB_BOTTOM_FMT = (GPU_FB_BOTTOM_FMT & ~7) | GSP_RGB565_OES;
+    GPU_FB_BOTTOM_FMT = format;
     GPU_FB_BOTTOM_STRIDE = 240 * 2;
 
     return framebufferCacheSize;
@@ -205,7 +220,7 @@ void Draw_RestoreFramebuffer(void)
 
 void Draw_FlushFramebuffer(void)
 {
-    svcFlushProcessDataCache(CUR_PROCESS_HANDLE, FB_BOTTOM_VRAM_ADDR, FB_BOTTOM_SIZE);
+    svcFlushProcessDataCache(CUR_PROCESS_HANDLE, (u32)FB_BOTTOM_VRAM_ADDR, FB_BOTTOM_SIZE);
 }
 
 u32 Draw_GetCurrentFramebufferAddress(bool top, bool left)
@@ -226,6 +241,21 @@ u32 Draw_GetCurrentFramebufferAddress(bool top, bool left)
     }
 }
 
+void Draw_GetCurrentScreenInfo(u32 *width, bool *is3d, bool top)
+{
+    if (top)
+    {
+        bool isNormal2d = (GPU_FB_TOP_FMT & BIT(6)) != 0;
+        *is3d = (GPU_FB_TOP_FMT & BIT(5)) != 0;
+        *width = !(*is3d) && !isNormal2d ? 800 : 400;
+    }
+    else
+    {
+        *is3d = false;
+        *width = 320;
+    }
+}
+
 static inline void Draw_WriteUnaligned(u8 *dst, u32 tmp, u32 size)
 {
     memcpy(dst, &tmp, size);
@@ -313,6 +343,7 @@ static inline void Draw_ConvertPixelToBGR8(u8 *dst, const u8 *src, GSPGPU_Frameb
 
 typedef struct FrameBufferConvertArgs {
     u8 *buf;
+    u32 width;
     u8 startingLine;
     u8 numLines;
     bool top;
@@ -324,7 +355,7 @@ static void Draw_ConvertFrameBufferLinesKernel(const FrameBufferConvertArgs *arg
     static const u8 formatSizes[] = { 4, 3, 2, 2, 2 };
 
     GSPGPU_FramebufferFormat fmt = args->top ? (GSPGPU_FramebufferFormat)(GPU_FB_TOP_FMT & 7) : (GSPGPU_FramebufferFormat)(GPU_FB_BOTTOM_FMT & 7);
-    u32 width = args->top ? 400 : 320;
+    u32 width = args->width;
     u32 stride = args->top ? GPU_FB_TOP_STRIDE : GPU_FB_BOTTOM_STRIDE;
 
     u32 pa = Draw_GetCurrentFramebufferAddress(args->top, args->left);
@@ -340,8 +371,8 @@ static void Draw_ConvertFrameBufferLinesKernel(const FrameBufferConvertArgs *arg
     }
 }
 
-void Draw_ConvertFrameBufferLines(u8 *buf, u32 startingLine, u32 numLines, bool top, bool left)
+void Draw_ConvertFrameBufferLines(u8 *buf, u32 width, u32 startingLine, u32 numLines, bool top, bool left)
 {
-    FrameBufferConvertArgs args = { buf, (u8)startingLine, (u8)numLines, top, left };
+    FrameBufferConvertArgs args = { buf, width, (u8)startingLine, (u8)numLines, top, left };
     svcCustomBackdoor(Draw_ConvertFrameBufferLinesKernel, &args);
 }

sysmodules/rosalina/source/menu.c

@@ -214,14 +214,14 @@ void menuEnter(void)
         menuRefCount++;
         svcKernelSetState(0x10000, 1);
         svcSleepThread(5 * 1000 * 100LL);
-        if (Draw_AllocateFramebufferCache() == 0)
+        if (R_FAILED(Draw_AllocateFramebufferCache(FB_BOTTOM_SIZE)))
         {
             // Oops
             menuRefCount = 0;
             svcKernelSetState(0x10000, 1);
             svcSleepThread(5 * 1000 * 100LL);
         }
-
+        else
             Draw_SetupFramebuffer();
     }
     Draw_Unlock();

sysmodules/rosalina/source/menus.c

@@ -293,18 +293,20 @@ void RosalinaMenu_PowerOff(void) // Soft shutdown.
 static s64 timeSpentConvertingScreenshot = 0;
 static s64 timeSpentWritingScreenshot = 0;
 
-static Result RosalinaMenu_WriteScreenshot(IFile *file, bool top, bool left)
+static Result RosalinaMenu_WriteScreenshot(IFile *file, u32 width, bool top, bool left)
 {
     u64 total;
     Result res = 0;
-    u32 dimX = top ? 400 : 320;
+    u32 lineSize = 3 * width;
-    u32 lineSize = 3 * dimX;
     u32 remaining = lineSize * 240;
+
+    TRY(Draw_AllocateFramebufferCacheForScreenshot(remaining));
+
     u8 *framebufferCache = (u8 *)Draw_GetFramebufferCache();
     u8 *framebufferCacheEnd = framebufferCache + Draw_GetFramebufferCacheSize();
 
     u8 *buf = framebufferCache;
-    Draw_CreateBitmapHeader(framebufferCache, dimX, 240);
+    Draw_CreateBitmapHeader(framebufferCache, width, 240);
     buf += 54;
 
     u32 y = 0;
@@ -315,7 +317,7 @@ static Result RosalinaMenu_WriteScreenshot(IFile *file, bool top, bool left)
         u32 available = (u32)(framebufferCacheEnd - buf);
         u32 size = available < remaining ? available : remaining;
         u32 nlines = size / lineSize;
-        Draw_ConvertFrameBufferLines(buf, y, nlines, top, left);
+        Draw_ConvertFrameBufferLines(buf, width, y, nlines, top, left);
 
         s64 t1 = svcGetSystemTick();
         timeSpentConvertingScreenshot += t1 - t0;
@@ -326,7 +328,10 @@ static Result RosalinaMenu_WriteScreenshot(IFile *file, bool top, bool left)
         remaining -= lineSize * nlines;
         buf = framebufferCache;
     }
-    end: return res;
+    end:
+
+    Draw_FreeFramebufferCache();
+    return res;
 }
 
 void RosalinaMenu_TakeScreenshot(void)
@@ -350,9 +355,16 @@ void RosalinaMenu_TakeScreenshot(void)
     archiveId = isSdMode ? ARCHIVE_SDMC : ARCHIVE_NAND_RW;
     Draw_Lock();
     Draw_RestoreFramebuffer();
+    Draw_FreeFramebufferCache();
 
     svcFlushEntireDataCache();
 
+    bool is3d;
+    u32 topWidth, bottomWidth; // actually Y-dim
+
+    Draw_GetCurrentScreenInfo(&bottomWidth, &is3d, false);
+    Draw_GetCurrentScreenInfo(&topWidth, &is3d, true);
+
     res = FSUSER_OpenArchive(&archive, archiveId, fsMakePath(PATH_EMPTY, ""));
     if(R_SUCCEEDED(res))
     {
@@ -407,24 +419,28 @@ void RosalinaMenu_TakeScreenshot(void)
 
     sprintf(filename, "/luma/screenshots/%04lu-%02lu-%02lu_%02lu-%02lu-%02lu.%03llu_top.bmp", year, month, days, hours, minutes, seconds, milliseconds);
     TRY(IFile_Open(&file, archiveId, fsMakePath(PATH_EMPTY, ""), fsMakePath(PATH_ASCII, filename), FS_OPEN_CREATE | FS_OPEN_WRITE));
-    TRY(RosalinaMenu_WriteScreenshot(&file, true, true));
+    TRY(RosalinaMenu_WriteScreenshot(&file, topWidth, true, true));
     TRY(IFile_Close(&file));
 
     sprintf(filename, "/luma/screenshots/%04lu-%02lu-%02lu_%02lu-%02lu-%02lu.%03llu_bot.bmp", year, month, days, hours, minutes, seconds, milliseconds);
     TRY(IFile_Open(&file, archiveId, fsMakePath(PATH_EMPTY, ""), fsMakePath(PATH_ASCII, filename), FS_OPEN_CREATE | FS_OPEN_WRITE));
-    TRY(RosalinaMenu_WriteScreenshot(&file, false, true));
+    TRY(RosalinaMenu_WriteScreenshot(&file, bottomWidth, false, true));
     TRY(IFile_Close(&file));
 
-    if((GPU_FB_TOP_FMT & 0x20) && (Draw_GetCurrentFramebufferAddress(true, true) != Draw_GetCurrentFramebufferAddress(true, false)))
+    if(is3d && (Draw_GetCurrentFramebufferAddress(true, true) != Draw_GetCurrentFramebufferAddress(true, false)))
     {
         sprintf(filename, "/luma/screenshots/%04lu-%02lu-%02lu_%02lu-%02lu-%02lu.%03llu_top_right.bmp", year, month, days, hours, minutes, seconds, milliseconds);
         TRY(IFile_Open(&file, archiveId, fsMakePath(PATH_EMPTY, ""), fsMakePath(PATH_ASCII, filename), FS_OPEN_CREATE | FS_OPEN_WRITE));
-        TRY(RosalinaMenu_WriteScreenshot(&file, true, false));
+        TRY(RosalinaMenu_WriteScreenshot(&file, topWidth, true, false));
         TRY(IFile_Close(&file));
     }
 
 end:
     IFile_Close(&file);
+
+    if (R_FAILED(Draw_AllocateFramebufferCache(FB_BOTTOM_SIZE)))
+        __builtin_trap(); // We're f***ed if this happens
+
     svcFlushEntireDataCache();
     Draw_SetupFramebuffer();
     Draw_Unlock();