Skip to content

Commit

Permalink
usb: add extra cmds for extracted FS handling
Browse files Browse the repository at this point in the history 
Implements both `StartExtractedFsDump` and `EndExtractedFsDump` commands into both nxdumptool and the Python host script.

Other changes include:

* poc: wake the write thread up if a preprocessing error occurs in all extracted FS dump functions. Fixes previously unhandled hangups.
* poc: verify current NCA hash before sending its last data chunk while dumping a NSP.

* host: update command handler to support CancelFileTransfer commands issued in-between SendFileProperties commands.
* host: update Markdown document.

* usb: use UsbCommandType_Count for the safety check in usbPrepareCommandHeader().
* usb: log both USB command header and command block whenever an error is reported by the host side.
  • Loading branch information
@DarkMatterCore
DarkMatterCore committed Nov 11, 2023
1 parent 95b6031 commit dcd1f66
Show file tree
Hide file tree
Showing 5 changed files with 252 additions and 65 deletions.
77 changes: 59 additions & 18 deletions code_templates/nxdt_rw_poc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3756,17 +3756,27 @@ static void extractedHfsReadThreadFunc(void *arg)
{
consolePrint("failed to retrieve free space from selected device\n");
shared_thread_data->read_error = true;
goto end;
}

if (shared_thread_data->total_size >= free_space)
if (!shared_thread_data->read_error && shared_thread_data->total_size >= free_space)
{
consolePrint("dump size exceeds free space\n");
shared_thread_data->read_error = true;
goto end;
}
} else {
if (!usbStartExtractedFsDump(shared_thread_data->total_size, filename))
{
consolePrint("failed to send extracted fs info to host\n");
shared_thread_data->read_error = true;
}
}

if (shared_thread_data->read_error)
{
condvarWakeAll(&g_writeCondvar);
goto end;
}

/* Loop through all file entries. */
for(u32 i = 0; i < hfs_entry_count; i++)
{
Expand Down Expand Up @@ -3912,6 +3922,8 @@ static void extractedHfsReadThreadFunc(void *arg)
if (shared_thread_data->data_size) condvarWait(&g_readCondvar, &g_fileMutex);
mutexUnlock(&g_fileMutex);

if (dev_idx == 1) usbEndExtractedFsDump();

consolePrint("successfully saved extracted hfs partition data to \"%s\"\n", filename);
consoleRefresh();
}
Expand Down Expand Up @@ -4131,15 +4143,25 @@ static void extractedPartitionFsReadThreadFunc(void *arg)
{
consolePrint("failed to retrieve free space from selected device\n");
shared_thread_data->read_error = true;
goto end;
}

if (shared_thread_data->total_size >= free_space)
if (!shared_thread_data->read_error && shared_thread_data->total_size >= free_space)
{
consolePrint("dump size exceeds free space\n");
shared_thread_data->read_error = true;
goto end;
}
} else {
if (!usbStartExtractedFsDump(shared_thread_data->total_size, filename))
{
consolePrint("failed to send extracted fs info to host\n");
shared_thread_data->read_error = true;
}
}

if (shared_thread_data->read_error)
{
condvarWakeAll(&g_writeCondvar);
goto end;
}

/* Loop through all file entries. */
Expand Down Expand Up @@ -4287,6 +4309,8 @@ static void extractedPartitionFsReadThreadFunc(void *arg)
if (shared_thread_data->data_size) condvarWait(&g_readCondvar, &g_fileMutex);
mutexUnlock(&g_fileMutex);

if (dev_idx == 1) usbEndExtractedFsDump();

consolePrint("successfully saved extracted partitionfs section data to \"%s\"\n", filename);
consoleRefresh();
}
Expand Down Expand Up @@ -4437,15 +4461,25 @@ static void extractedRomFsReadThreadFunc(void *arg)
{
consolePrint("failed to retrieve free space from selected device\n");
shared_thread_data->read_error = true;
goto end;
}

if (shared_thread_data->total_size >= free_space)
if (!shared_thread_data->read_error && shared_thread_data->total_size >= free_space)
{
consolePrint("dump size exceeds free space\n");
shared_thread_data->read_error = true;
goto end;
}
} else {
if (!usbStartExtractedFsDump(shared_thread_data->total_size, filename))
{
consolePrint("failed to send extracted fs info to host\n");
shared_thread_data->read_error = true;
}
}

if (shared_thread_data->read_error)
{
condvarWakeAll(&g_writeCondvar);
goto end;
}

/* Reset current file table offset. */
Expand Down Expand Up @@ -4601,6 +4635,8 @@ static void extractedRomFsReadThreadFunc(void *arg)
if (shared_thread_data->data_size) condvarWait(&g_readCondvar, &g_fileMutex);
mutexUnlock(&g_fileMutex);

if (dev_idx == 1) usbEndExtractedFsDump();

consolePrint("successfully saved extracted romfs section data to \"%s\"\n", filename);
consoleRefresh();
}
Expand Down Expand Up @@ -5293,6 +5329,19 @@ static void nspThreadFunc(void *arg)
// update clean hash calculation
sha256ContextUpdate(&clean_sha256_ctx, buf, blksize);

if ((offset + blksize) >= cur_nca_ctx->content_size)
{
// get clean hash
sha256ContextGetHash(&clean_sha256_ctx, clean_sha256_hash);

// validate clean hash
if (!cnmtVerifyContentHash(&cnmt_ctx, cur_nca_ctx, clean_sha256_hash))
{
consolePrint("sha256 checksum mismatch for nca \"%s\"\n", cur_nca_ctx->content_id_str);
goto end;
}
}

if (dirty_header)
{
// write re-encrypted headers
Expand Down Expand Up @@ -5334,17 +5383,9 @@ static void nspThreadFunc(void *arg)
}
}

// get hashes
sha256ContextGetHash(&clean_sha256_ctx, clean_sha256_hash);
// get dirty hash
sha256ContextGetHash(&dirty_sha256_ctx, dirty_sha256_hash);

// verify content hash
if (!cnmtVerifyContentHash(&cnmt_ctx, cur_nca_ctx, clean_sha256_hash))
{
consolePrint("sha256 checksum mismatch for nca \"%s\"\n", cur_nca_ctx->content_id_str);
goto end;
}

if (memcmp(clean_sha256_hash, dirty_sha256_hash, SHA256_HASH_SIZE) != 0)
{
// update content id and hash
Expand Down
55 changes: 43 additions & 12 deletions host/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ Unless stated otherwise, the reader must assume all integer fields in the docume
* [CancelFileTransfer](#cancelfiletransfer).
* [SendNspHeader](#sendnspheader).
* [EndSession](#endsession).
* [StartExtractedFsDump](#startextractedfsdump).
* [EndExtractedFsDump](#endextractedfsdump).
* [Status response](#status-response).
* [Status codes](#status-codes).
* [NSP transfer mode](#nsp-transfer-mode).
Expand Down Expand Up @@ -101,13 +103,15 @@ Certain commands yield no command block at all, leading to a command block size

#### Command IDs

| Value | Name | Description |
|-------|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
| 0 | [`StartSession`](#startsession) | Starts a USB session between the target console and the USB host device. |
| 1 | [`SendFileProperties`](#sendfileproperties) | Sends file metadata and starts a data transfer process. |
| 2 | [`CancelFileTransfer`](#cancelfiletransfer) | Cancels an ongoing data transfer process started by a previously issued [`SendFileProperties`](#sendfileproperties) command. |
| 3 | [`SendNspHeader`](#sendnspheader) | Sends the `PFS0` header from a Nintendo Submission Package (NSP). Only issued under [NSP transfer mode](#nsp-transfer-mode). |
| 4 | [`EndSession`](#endsession) | Ends a previously stablished USB session between the target console and the USB host device. |
| Value | Name | Description |
|-------|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
| 0 | [`StartSession`](#startsession) | Starts a USB session between the target console and the USB host device. |
| 1 | [`SendFileProperties`](#sendfileproperties) | Sends file metadata and starts a data transfer process. |
| 2 | [`CancelFileTransfer`](#cancelfiletransfer) | Cancels an ongoing data transfer process started by a previously issued [`SendFileProperties`](#sendfileproperties) command. |
| 3 | [`SendNspHeader`](#sendnspheader) | Sends the `PFS0` header from a Nintendo Submission Package (NSP). Only issued under [NSP transfer mode](#nsp-transfer-mode). |
| 4 | [`EndSession`](#endsession) | Ends a previously stablished USB session between the target console and the USB host device. |
| 5 | [`StartExtractedFsDump`](#startextractedfsdump) | Informs the host device that an extracted filesystem dump (e.g. HFS, PFS, RomFS) is about to begin. |
| 6 | [`EndExtractedFsDump`](#endextractedfsdump) | Informs the host device that a previously started filesystem dump (via [`StartExtractedFsDump`](#startextractedfsdump)) has finished. |

### Command blocks

Expand All @@ -134,11 +138,11 @@ Size: 0x320 bytes.

| Offset | Size | Type | Description |
|--------|-------|---------------|----------------------------------------------|
| 0x000 | 0x08 | `uint64_t` | File size. |
| 0x008 | 0x04 | `uint32_t` | Path length. |
| 0x00C | 0x04 | `uint32_t` | [NSP header size](#nsp-transfer-mode). |
| 0x000 | 0x008 | `uint64_t` | File size. |
| 0x008 | 0x004 | `uint32_t` | Path length. |
| 0x00C | 0x004 | `uint32_t` | [NSP header size](#nsp-transfer-mode). |
| 0x010 | 0x301 | `char[769]` | UTF-8 encoded path (NULL-terminated string). |
| 0x311 | 0x0F | `uint8_t[15]` | Reserved. |
| 0x311 | 0x00F | `uint8_t[15]` | Reserved. |

Sent right before starting a file transfer. If it succeeds, a data transfer stage will take place using 8 MiB (0x800000) chunks. If needed, the last chunk will be truncated.

Expand All @@ -158,7 +162,12 @@ Finally, it should be noted that it's possible for the `filesize` field to be ze

Yields no command block. Expects a status response, just like the rest of the commands.

This command can only be issued during the file data transfer stage from a [SendFileProperties](#sendfileproperties) command. It is used to gracefully cancel an ongoing file transfer while also keeping the USB session alive. It's up to the USB host to decide what to do with the incomplete data.
This command can only be issued under two different scenarios:

* During the file data transfer stage from a [SendFileProperties](#sendfileproperties) command.
* In-between two different [SendFileProperties](#sendfileproperties) commands while under [NSP transfer mode](#nsp-transfer-mode).

It is used to gracefully cancel an ongoing file transfer while also keeping the USB session alive. It's up to the USB host to decide what to do with the incomplete data.

The easiest way to detect this command during a file transfer is by checking the length of the last received block and then parse it to see if it matches a `CancelFileTransfer` command header.

Expand All @@ -176,6 +185,28 @@ Yields no command block. Expects a status response, just like the rest of the co

This command is only issued while exiting nxdumptool, as long as the target console is connected to a host device and a USB session has been successfully established.

#### StartExtractedFsDump

Size: 0x310 bytes.

| Offset | Size | Type | Description |
|--------|-------|---------------|----------------------------------------------------------------|
| 0x000 | 0x008 | `uint64_t` | Extracted FS dump size. |
| 0x008 | 0x301 | `char[769]` | UTF-8 encoded extracted FS root path (NULL-terminated string). |
| 0x309 | 0x006 | `uint8_t[6]` | Reserved. |

Sent right before dumping a Switch FS in extracted form (e.g. HFS, PFS, RomFS) using multiple [SendFileProperties](#sendfileproperties) commands in succession.

The extracted FS dump size field can be used by the host device to calculate an ETA for the overall FS dump.

The extracted FS root path represents a path relative to the output directory where all the extracted FS entries are stored. All file paths from the extracted FS dump will begin with this string.

#### EndExtractedFsDump

Yields no command block. Expects a status response, just like the rest of the commands.

This command is only issued after all file entries from an extracted FS dump (started via [`StartExtractedFsDump`](#startextractedfsdump)) have been successfully transferred to the host device.

### Status response

Size: 0x10 bytes.
Expand Down
83 changes: 64 additions & 19 deletions host/nxdt_host.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
USB_DEV_PRODUCT = 'nxdumptool'

# USB timeout (milliseconds).
USB_TRANSFER_TIMEOUT = 5000
USB_TRANSFER_TIMEOUT = 10000

# USB transfer block size.
USB_TRANSFER_BLOCK_SIZE = 0x800000
Expand All @@ -102,15 +102,18 @@
USB_CMD_HEADER_SIZE = 0x10

# USB command IDs.
USB_CMD_START_SESSION = 0
USB_CMD_SEND_FILE_PROPERTIES = 1
USB_CMD_CANCEL_FILE_TRANSFER = 2
USB_CMD_SEND_NSP_HEADER = 3
USB_CMD_END_SESSION = 4
USB_CMD_START_SESSION = 0
USB_CMD_SEND_FILE_PROPERTIES = 1
USB_CMD_CANCEL_FILE_TRANSFER = 2
USB_CMD_SEND_NSP_HEADER = 3
USB_CMD_END_SESSION = 4
USB_CMD_START_EXTRACTED_FS_DUMP = 5
USB_CMD_END_EXTRACTED_FS_DUMP = 6

# USB command block sizes.
USB_CMD_BLOCK_SIZE_START_SESSION = 0x10
USB_CMD_BLOCK_SIZE_SEND_FILE_PROPERTIES = 0x320
USB_CMD_BLOCK_SIZE_START_SESSION = 0x10
USB_CMD_BLOCK_SIZE_SEND_FILE_PROPERTIES = 0x320
USB_CMD_BLOCK_SIZE_START_EXTRACTED_FS_DUMP = 0x310

# Max filename length (file properties).
USB_FILE_PROPERTIES_MAX_NAME_LENGTH = 0x300
Expand Down Expand Up @@ -567,9 +570,14 @@ def utilsGetPath(path_arg: str, fallback_path: str, is_file: bool, create: bool
def utilsIsValueAlignedToEndpointPacketSize(value: int) -> bool:
return bool((value & (g_usbEpMaxPacketSize - 1)) == 0)

def utilsResetNspInfo() -> None:
def utilsResetNspInfo(delete: bool = False) -> None:
global g_nspTransferMode, g_nspSize, g_nspHeaderSize, g_nspRemainingSize, g_nspFile, g_nspFilePath

if g_nspFile:
g_nspFile.close()
if delete:
os.remove(g_nspFilePath)

# Reset NSP transfer mode info.
g_nspTransferMode = False
g_nspSize = 0
Expand Down Expand Up @@ -868,9 +876,7 @@ def usbHandleSendFileProperties(cmd_block: bytes) -> int | None:

def cancelTransfer():
# Cancel file transfer.
file.close()
os.remove(fullpath)
utilsResetNspInfo()
utilsResetNspInfo(True)
if use_pbar:
g_progressBarWindow.end()

Expand Down Expand Up @@ -941,6 +947,19 @@ def cancelTransfer():

return USB_STATUS_SUCCESS

def usbHandleCancelFileTransfer(cmd_block: bytes) -> int:
#assert g_logger is not None

g_logger.debug(f'Received CancelFileTransfer ({USB_CMD_START_SESSION:02X}) command.')

if g_nspTransferMode:
utilsResetNspInfo(True)
g_logger.warning('Transfer cancelled.')
return USB_STATUS_SUCCESS
else:
g_logger.error('Unexpected transfer cancellation.')
return USB_STATUS_MALFORMED_CMD

def usbHandleSendNspHeader(cmd_block: bytes) -> int:
global g_nspTransferMode, g_nspHeaderSize, g_nspRemainingSize, g_nspFile, g_nspFilePath

Expand All @@ -967,7 +986,6 @@ def usbHandleSendNspHeader(cmd_block: bytes) -> int:
# Write NSP header.
g_nspFile.seek(0)
g_nspFile.write(cmd_block)
g_nspFile.close()

g_logger.debug(f'Successfully wrote 0x{nsp_header_size:X}-byte long NSP header to "{g_nspFilePath}".\n')

Expand All @@ -981,15 +999,41 @@ def usbHandleEndSession(cmd_block: bytes) -> int:
g_logger.debug(f'Received EndSession ({USB_CMD_END_SESSION:02X}) command.')
return USB_STATUS_SUCCESS

def usbHandleStartExtractedFsDump(cmd_block: bytes) -> int:
#assert g_logger is not None

g_logger.debug(f'Received StartExtractedFsDump ({USB_CMD_START_EXTRACTED_FS_DUMP:02X}) command.')

if g_nspTransferMode:
g_logger.error('StartExtractedFsDump received mid NSP transfer.')
return USB_STATUS_MALFORMED_CMD

# Parse command block.
(extracted_fs_size, extracted_fs_root_path) = struct.unpack_from(f'<Q{USB_FILE_PROPERTIES_MAX_NAME_LENGTH}s', cmd_block, 0)
extracted_fs_root_path = extracted_fs_root_path.decode('utf-8').strip('\x00')

g_logger.info(f'Starting extracted FS dump (size 0x{extracted_fs_size:X}, output relative path "{extracted_fs_root_path}").')

# Return status code.
return USB_STATUS_SUCCESS

def usbHandleEndExtractedFsDump(cmd_block: bytes) -> int:
#assert g_logger is not None
g_logger.debug(f'Received EndExtractedFsDump ({USB_CMD_END_EXTRACTED_FS_DUMP:02X}) command.')
g_logger.info(f'Finished extracted FS dump.')
return USB_STATUS_SUCCESS

def usbCommandHandler() -> None:
#assert g_logger is not None

# CancelFileTransfer is handled in usbHandleSendFileProperties().
cmd_dict = {
USB_CMD_START_SESSION: usbHandleStartSession,
USB_CMD_SEND_FILE_PROPERTIES: usbHandleSendFileProperties,
USB_CMD_SEND_NSP_HEADER: usbHandleSendNspHeader,
USB_CMD_END_SESSION: usbHandleEndSession
USB_CMD_START_SESSION: usbHandleStartSession,
USB_CMD_SEND_FILE_PROPERTIES: usbHandleSendFileProperties,
USB_CMD_CANCEL_FILE_TRANSFER: usbHandleCancelFileTransfer,
USB_CMD_SEND_NSP_HEADER: usbHandleSendNspHeader,
USB_CMD_END_SESSION: usbHandleEndSession,
USB_CMD_START_EXTRACTED_FS_DUMP: usbHandleStartExtractedFsDump,
USB_CMD_END_EXTRACTED_FS_DUMP: usbHandleEndExtractedFsDump
}

# Get device endpoints.
Expand Down Expand Up @@ -1050,7 +1094,8 @@ def usbCommandHandler() -> None:
# Verify command block size.
if (cmd_id == USB_CMD_START_SESSION and cmd_block_size != USB_CMD_BLOCK_SIZE_START_SESSION) or \
(cmd_id == USB_CMD_SEND_FILE_PROPERTIES and cmd_block_size != USB_CMD_BLOCK_SIZE_SEND_FILE_PROPERTIES) or \
(cmd_id == USB_CMD_SEND_NSP_HEADER and not cmd_block_size):
(cmd_id == USB_CMD_SEND_NSP_HEADER and not cmd_block_size) or \
(cmd_id == USB_CMD_START_EXTRACTED_FS_DUMP and cmd_block_size != USB_CMD_BLOCK_SIZE_START_EXTRACTED_FS_DUMP):
g_logger.error(f'Invalid command block size for command ID {cmd_id:02X}! (0x{cmd_block_size:X}).\n')
usbSendStatus(USB_STATUS_MALFORMED_CMD)
continue
Expand Down
Loading

0 comments on commit dcd1f66

Please sign in to comment.