From ca49956219d5f505de2555210fbc3cdf293a8263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Lam?= Date: Sun, 24 Nov 2019 11:38:02 +0100 Subject: [PATCH 1/2] loader: Compile C++ code with -std=gnu++17 --- sysmodules/loader/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sysmodules/loader/Makefile b/sysmodules/loader/Makefile index 14c390d..c24cde7 100755 --- a/sysmodules/loader/Makefile +++ b/sysmodules/loader/Makefile @@ -28,13 +28,13 @@ INCLUDES := include ARCH := -march=armv6k -mtune=mpcore -mfloat-abi=hard -mtp=soft DEFINES := -DARM11 -D_3DS -CFLAGS := -g -std=gnu11 -Wall -Wextra -Werror -O2 -mword-relocations \ +COMMON_FLAGS = -g -Wall -Wextra -Werror -O2 -mword-relocations \ -fomit-frame-pointer -ffunction-sections -fdata-sections \ - $(ARCH) $(DEFINES) + $(ARCH) $(DEFINES) $(INCLUDE) -CFLAGS += $(INCLUDE) +CFLAGS := -std=gnu11 $(COMMON_FLAGS) -CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions -std=gnu++11 +CXXFLAGS := -fno-rtti -fno-exceptions -std=gnu++17 $(COMMON_FLAGS) ASFLAGS := -g $(ARCH) LDFLAGS = -specs=3dsx.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map),--section-start,.text=0x14000000 From 8233d4e2267041366d06bdd2ae6369bc57a5de26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Lam?= Date: Mon, 8 Jul 2019 18:28:35 +0200 Subject: [PATCH 2/2] loader: Add support for BPS patches The BPS format allows distributing patches that are smaller and that do not contain copyrighted content if data is relocated (unlike non-trivial IPS patches). This is essential for games such as Majora's Mask 3D that have three barely different code revisions. Supporting all three versions would demand an unreasonable amount of work; with BPS patches only one version has to be supported. The patcher is written in C++ in order to make it possible to share the implementation with Citra and because a C version would be a lot more ugly and tedious to write. The patcher is non-intrusive for the rest of the codebase and self-contained so hopefully that isn't an issue. This adds roughly ~0x500 bytes of code to the loader sysmodule. Code looks reasonably optimised (in IDA). Tested and works on an o3DS. --- sysmodules/loader/source/bps_patcher.cpp | 283 +++++++++++++++++++++++ sysmodules/loader/source/bps_patcher.h | 12 + sysmodules/loader/source/file_util.h | 76 ++++++ sysmodules/loader/source/patcher.c | 2 + 4 files changed, 373 insertions(+) create mode 100644 sysmodules/loader/source/bps_patcher.cpp create mode 100644 sysmodules/loader/source/bps_patcher.h create mode 100644 sysmodules/loader/source/file_util.h diff --git a/sysmodules/loader/source/bps_patcher.cpp b/sysmodules/loader/source/bps_patcher.cpp new file mode 100644 index 0000000..0c8ca0a --- /dev/null +++ b/sysmodules/loader/source/bps_patcher.cpp @@ -0,0 +1,283 @@ +#include "bps_patcher.h" + +#include +#include +#include +#include +#include + +extern "C" +{ +#include <3ds/os.h> +#include <3ds/result.h> +#include <3ds/services/fs.h> +#include <3ds/svc.h> + +#include "patcher.h" +#include "strings.h" +} + +#include "file_util.h" + +namespace patcher +{ +namespace Bps +{ +// The BPS format uses variable length encoding for all integers. +// Realistically uint32s are more than enough for code patching. +using Number = u32; + +constexpr std::size_t FooterSize = 12; + +// The BPS format uses CRC32 checksums. +[[gnu::optimize("Os")]] static u32 crc32(const u8 *data, std::size_t size) +{ + u32 crc = 0xFFFFFFFF; + for(std::size_t i = 0; i < size; ++i) + { + crc ^= data[i]; + for(std::size_t j = 0; j < 8; ++j) + { + u32 mask = -(crc & 1); + crc = (crc >> 1) ^ (0xEDB88320 & mask); + } + } + return ~crc; +} + +// Utility class to make keeping track of offsets and bound checks less error prone. +template +class Stream +{ +public: + Stream(T *ptr, std::size_t size) : m_ptr{ptr}, m_size{size} {} + + bool Read(void *buffer, std::size_t length) + { + if(m_offset + length > m_size) + return false; + std::memcpy(buffer, m_ptr + m_offset, length); + m_offset += length; + return true; + } + + template + [[gnu::optimize("Os")]] bool CopyFrom(Stream &other, std::size_t length) + { + if(m_offset + length > m_size) + return false; + if(!other.Read(m_ptr + m_offset, length)) + return false; + m_offset += length; + return true; + } + + template + std::optional Read() + { + static_assert(std::is_pod_v); + ValueType val{}; + if(!Read(&val, sizeof(val))) + return std::nullopt; + return val; + } + + [[gnu::optimize("Os")]] Number ReadNumber() + { + Number data = 0, shift = 1; + std::optional x; + while((x = Read())) + { + data += (*x & 0x7f) * shift; + if(*x & 0x80) + break; + shift <<= 7; + data += shift; + } + return data; + } + + auto data() const { return m_ptr; } + std::size_t size() const { return m_size; } + std::size_t Tell() const { return m_offset; } + + bool Seek(size_t offset) + { + m_offset = offset; + return true; + } + +private: + T *m_ptr = nullptr; + std::size_t m_size = 0; + std::size_t m_offset = 0; +}; + +class PatchApplier +{ +public: + PatchApplier(Stream source, Stream target, Stream patch) + : m_source{source}, m_target{target}, m_patch{patch} + { + } + + [[gnu::always_inline]] bool Apply() + { + const auto magic = *m_patch.Read>(); + if(std::string_view(magic.data(), magic.size()) != "BPS1") + return false; + + const Bps::Number source_size = m_patch.ReadNumber(); + const Bps::Number target_size = m_patch.ReadNumber(); + const Bps::Number metadata_size = m_patch.ReadNumber(); + if(source_size > m_source.size() || target_size > m_target.size() || metadata_size != 0) + return false; + + const std::size_t command_start_offset = m_patch.Tell(); + const std::size_t command_end_offset = m_patch.size() - FooterSize; + m_patch.Seek(command_end_offset); + const u32 source_crc32 = *m_patch.Read(); + const u32 target_crc32 = *m_patch.Read(); + m_patch.Seek(command_start_offset); + + if(crc32(m_source.data(), source_size) != source_crc32) + return false; + + // Process all patch commands. + std::memset(m_target.data(), 0, m_target.size()); + while(m_patch.Tell() < command_end_offset) + { + const bool ok = HandleCommand(); + if(!ok) + return false; + } + + return crc32(m_target.data(), target_size) == target_crc32; + } + +private: + bool HandleCommand() + { + const Number data = m_patch.ReadNumber(); + const Number command = data & 3; + const Number length = (data >> 2) + 1; + + switch(command) + { + case 0: + return SourceRead(length); + case 1: + return TargetRead(length); + case 2: + return SourceCopy(length); + case 3: + return TargetCopy(length); + default: + return false; + } + } + + bool SourceRead(Number length) + { + return m_source.Seek(m_target.Tell()) && m_target.CopyFrom(m_source, length); + } + + bool TargetRead(Number length) { return m_target.CopyFrom(m_patch, length); } + + bool SourceCopy(Number length) + { + const Number data = m_patch.ReadNumber(); + m_source_relative_offset += (data & 1 ? -1 : +1) * int(data >> 1); + if(!m_source.Seek(m_source_relative_offset) || !m_target.CopyFrom(m_source, length)) + return false; + m_source_relative_offset += length; + return true; + } + + bool TargetCopy(Number length) + { + const Number data = m_patch.ReadNumber(); + m_target_relative_offset += (data & 1 ? -1 : +1) * int(data >> 1); + if(m_target.Tell() + length > m_target.size()) + return false; + if(m_target_relative_offset + length > m_target.size()) + return false; + // Byte by byte copy. + for(size_t i = 0; i < length; ++i) + m_target.data()[m_target.Tell() + i] = m_target.data()[m_target_relative_offset++]; + m_target.Seek(m_target.Tell() + length); + return true; + } + + std::size_t m_source_relative_offset = 0; + std::size_t m_target_relative_offset = 0; + Stream m_source; + Stream m_target; + Stream m_patch; +}; + +} // namespace Bps + +class ScopedAppHeap +{ +public: + ScopedAppHeap() + { + u32 tmp; + m_size = osGetMemRegionFree(MEMREGION_APPLICATION); + if(!R_SUCCEEDED(svcControlMemory(&tmp, BaseAddress, 0, m_size, + MemOp(MEMOP_ALLOC | MEMOP_REGION_APP), + MemPerm(MEMPERM_READ | MEMPERM_WRITE)))) + { + svcBreak(USERBREAK_PANIC); + } + } + + ~ScopedAppHeap() + { + u32 tmp; + svcControlMemory(&tmp, BaseAddress, 0, m_size, MEMOP_FREE, MemPerm(0)); + } + + static constexpr u32 BaseAddress = 0x08000000; + +private: + u32 m_size; +}; + +static inline bool ApplyCodeBpsPatch(u64 prog_id, u8 *code, u32 size) +{ + char bps_path[] = "/luma/titles/0000000000000000/code.bps"; + progIdToStr(bps_path + 28, prog_id); + util::File patch_file; + if(!patch_file.Open(bps_path, FS_OPEN_READ)) + return true; + const u32 patch_size = u32(patch_file.GetSize().value_or(0)); + + // Temporarily use APPLICATION memory to store the source and patch data. + ScopedAppHeap memory; + + u8 *source_data = reinterpret_cast(memory.BaseAddress); + u8 *patch_data = source_data + size; + std::memcpy(source_data, code, size); + if(!patch_file.Read(patch_data, patch_size, 0)) + return false; + + Bps::Stream source_stream{source_data, size}; + Bps::Stream target_stream{code, size}; + Bps::Stream patch_stream{patch_data, patch_size}; + Bps::PatchApplier applier{source_stream, target_stream, patch_stream}; + if(!applier.Apply()) + svcBreak(USERBREAK_PANIC); + return true; +} + +} // namespace patcher + +extern "C" +{ + bool patcherApplyCodeBpsPatch(u64 progId, u8 *code, u32 size) + { + return patcher::ApplyCodeBpsPatch(progId, code, size); + } +} diff --git a/sysmodules/loader/source/bps_patcher.h b/sysmodules/loader/source/bps_patcher.h new file mode 100644 index 0000000..25e76b0 --- /dev/null +++ b/sysmodules/loader/source/bps_patcher.h @@ -0,0 +1,12 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif +#include <3ds/types.h> + +bool patcherApplyCodeBpsPatch(u64 progId, u8* code, u32 size); + +#ifdef __cplusplus +} +#endif diff --git a/sysmodules/loader/source/file_util.h b/sysmodules/loader/source/file_util.h new file mode 100644 index 0000000..21aee71 --- /dev/null +++ b/sysmodules/loader/source/file_util.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + +extern "C" +{ +#include <3ds/result.h> +#include <3ds/services/fs.h> +#include <3ds/svc.h> +#include <3ds/types.h> +} + +namespace util +{ +inline FS_Path MakePath(const char *path) +{ + return {PATH_ASCII, strnlen(path, 255) + 1, path}; +} + +// A small wrapper to make forgetting to close a file and +// to check read lengths impossible. +class File +{ +public: + File() = default; + File(const File &other) = delete; + File &operator=(const File &) = delete; + File(File &&other) { *this = std::move(other); } + File &operator=(File &&other) + { + std::swap(m_handle, other.m_handle); + return *this; + } + + ~File() { Close(); } + + bool Close() + { + const bool ok = !m_handle || R_SUCCEEDED(FSFILE_Close(*m_handle)); + if(ok) + m_handle = std::nullopt; + return ok; + } + + bool Open(const char *path, int open_flags) + { + const FS_Path archive_path = {PATH_EMPTY, 1, ""}; + Handle handle; + const bool ok = R_SUCCEEDED(FSUSER_OpenFileDirectly(&handle, ARCHIVE_SDMC, archive_path, + MakePath(path), open_flags, 0)); + if(ok) + m_handle = handle; + return ok; + } + + bool Read(void *buffer, u32 size, u64 offset) + { + u32 bytes_read = 0; + const Result res = FSFILE_Read(*m_handle, &bytes_read, offset, buffer, size); + return R_SUCCEEDED(res) && bytes_read == size; + } + + std::optional GetSize() const + { + u64 size; + if(!R_SUCCEEDED(FSFILE_GetSize(*m_handle, &size))) + return std::nullopt; + return size; + } + +private: + std::optional m_handle; +}; + +} // namespace util diff --git a/sysmodules/loader/source/patcher.c b/sysmodules/loader/source/patcher.c index 8884106..7967cdd 100644 --- a/sysmodules/loader/source/patcher.c +++ b/sysmodules/loader/source/patcher.c @@ -1,5 +1,6 @@ #include <3ds.h> #include "patcher.h" +#include "bps_patcher.h" #include "memory.h" #include "strings.h" #include "romfsredir.h" @@ -845,6 +846,7 @@ void patchCode(u64 progId, u16 progVer, u8 *code, u32 size, u32 textSize, u32 ro if(CONFIG(PATCHGAMES)) { + if(!patcherApplyCodeBpsPatch(progId, code, size)) goto error; if(!applyCodeIpsPatch(progId, code, size)) goto error; if((u32)((progId >> 0x20) & 0xFFFFFFEDULL) == 0x00040000)