From 6e2884319b189b25fae3f9a17238b2d92c5f9c51 Mon Sep 17 00:00:00 2001 From: Jeffrey Knockel Date: Tue, 13 Feb 2024 08:36:21 -0500 Subject: [PATCH 1/2] libvncclient: support more tight 32bpp formats The tight decoder now supports the following 32bpp pixel formats in both big-endian and little-endian byte orders: ABGR, BGRA, ARGB, and RGBA. Prior to this change the supported 32bpp pixel formats were more limited and dependant on the requested byte order of the client's pixel format. --- src/libvncclient/tight.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libvncclient/tight.c b/src/libvncclient/tight.c index 57a3715b..2abfd0c5 100644 --- a/src/libvncclient/tight.c +++ b/src/libvncclient/tight.c @@ -635,10 +635,17 @@ DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h) pitch = w * pixelSize; dst = (uint8_t *)client->buffer; #else - if (client->format.bigEndian) flags |= TJ_ALPHAFIRST; - if (client->format.redShift == 16 && client->format.blueShift == 0) - flags |= TJ_BGR; - if (client->format.bigEndian) flags ^= TJ_BGR; + if (client->format.redShift == 24 && + client->format.greenShift == 16 && client->format.blueShift == 8) { + flags |= TJ_ALPHAFIRST | TJ_BGR; /* ABGR */ + } else if (client->format.redShift == 16 && + client->format.greenShift == 8 && client->format.blueShift == 0) { + flags |= TJ_BGR; /* BGRA */ + } else if (client->format.redShift == 8 && + client->format.greenShift == 16 && client->format.blueShift == 24) { + flags |= TJ_ALPHAFIRST; /* ARGB */ + } /* else RGBA */ + if (client->format.bigEndian) flags ^= TJ_ALPHAFIRST | TJ_BGR; pixelSize = BPP / 8; pitch = client->width * pixelSize; dst = &client->frameBuffer[y * pitch + x * pixelSize]; From d2e0354e4a94b144aa082c2f7a87909e936738c1 Mon Sep 17 00:00:00 2001 From: Jeffrey Knockel Date: Tue, 13 Feb 2024 08:46:16 -0500 Subject: [PATCH 2/2] libvncclient: support all tight 32bpp formats Prior to this change, the only 32bpp pixel formats that the tight decoder supported were the ABGR, BGRA, ARGB, and RGBA 32bpp pixel formats. This is because these are the only RGB formats that TurboJPEG can directly output. To support all possible 32bpp pixel formats with all possible maximums, shifts, etc., we need to perform additional conversion after decompressing the JPEG. Thankfully the 16bpp code path is already doing this, so we can reuse it if the client requests a 32bpp pixel format that TurboJPEG cannot directly output. --- src/libvncclient/tight.c | 85 ++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/src/libvncclient/tight.c b/src/libvncclient/tight.c index 2abfd0c5..6b93a520 100644 --- a/src/libvncclient/tight.c +++ b/src/libvncclient/tight.c @@ -600,6 +600,9 @@ DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h) int compressedLen; uint8_t *compressedData, *dst; int pixelSize, pitch, flags = 0; +#if BPP == 32 + int convertingRequired = 0; +#endif compressedLen = (int)ReadCompactLen(client); if (compressedLen <= 0) { @@ -629,27 +632,40 @@ DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h) } } -#if BPP == 16 - flags = 0; - pixelSize = 3; - pitch = w * pixelSize; - dst = (uint8_t *)client->buffer; -#else - if (client->format.redShift == 24 && - client->format.greenShift == 16 && client->format.blueShift == 8) { - flags |= TJ_ALPHAFIRST | TJ_BGR; /* ABGR */ - } else if (client->format.redShift == 16 && - client->format.greenShift == 8 && client->format.blueShift == 0) { - flags |= TJ_BGR; /* BGRA */ - } else if (client->format.redShift == 8 && - client->format.greenShift == 16 && client->format.blueShift == 24) { - flags |= TJ_ALPHAFIRST; /* ARGB */ - } /* else RGBA */ - if (client->format.bigEndian) flags ^= TJ_ALPHAFIRST | TJ_BGR; - pixelSize = BPP / 8; - pitch = client->width * pixelSize; - dst = &client->frameBuffer[y * pitch + x * pixelSize]; +#if BPP == 32 + if (client->format.depth == 24 && client->format.redMax == 0xFF && + client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) { + /* TurboJPEG can directly output some 24-bit depth 32 bpp pixel formats */ + if (client->format.redShift == 24 && + client->format.greenShift == 16 && client->format.blueShift == 8) { + flags |= TJ_ALPHAFIRST | TJ_BGR; /* ABGR */ + } else if (client->format.redShift == 16 && + client->format.greenShift == 8 && client->format.blueShift == 0) { + flags |= TJ_BGR; /* BGRA */ + } else if (client->format.redShift == 8 && + client->format.greenShift == 16 && client->format.blueShift == 24) { + flags |= TJ_ALPHAFIRST; /* ARGB */ + } else if (client->format.redShift == 0 && + client->format.greenShift == 8 && client->format.blueShift == 16) { + flags |= 0; /* RGBA */ + } else { + convertingRequired = 1; /* TurboJPEG cannot output this 24-bit depth format */ + } + } else { + convertingRequired = 1; /* TurboJPEG cannot output formats with this color depth and these maximums */ + } + if (!convertingRequired) { + if (client->format.bigEndian) flags ^= TJ_ALPHAFIRST | TJ_BGR; + pixelSize = BPP / 8; + pitch = client->width * pixelSize; + dst = &client->frameBuffer[y * pitch + x * pixelSize]; + } else #endif + { + pixelSize = 3; /* RGB */ + pitch = w * pixelSize; + dst = (uint8_t *)client->buffer; + } if (tjDecompress(client->tjhnd, compressedData, (unsigned long)compressedLen, dst, w, pitch, h, pixelSize, flags)==-1) { @@ -660,23 +676,26 @@ DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h) free(compressedData); -#if BPP == 16 - pixelSize = BPP / 8; - pitch = client->width * pixelSize; - dst = &client->frameBuffer[y * pitch + x * pixelSize]; +#if BPP == 32 + if (convertingRequired) +#endif { - CARDBPP *dst16=(CARDBPP *)dst, *dst2; - char *src = client->buffer; - int i, j; - - for (j = 0; j < h; j++) { - for (i = 0, dst2 = dst16; i < w; i++, dst2++, src += 3) { - *dst2 = RGB24_TO_PIXEL(BPP, src[0], src[1], src[2]); + pixelSize = BPP / 8; + pitch = client->width * pixelSize; + dst = &client->frameBuffer[y * pitch + x * pixelSize]; + { + CARDBPP *dstbpp=(CARDBPP *)dst, *dst2; + char *src = client->buffer; + int i, j; + + for (j = 0; j < h; j++) { + for (i = 0, dst2 = dstbpp; i < w; i++, dst2++, src += 3) { + *dst2 = RGB24_TO_PIXEL(BPP, src[0], src[1], src[2]); + } + dstbpp += client->width; } - dst16 += client->width; } } -#endif return TRUE; }