Skip to content

Commit

Permalink
src,build: port CGDisplayStream capturing to ScreenCaptureKit
Browse files Browse the repository at this point in the history
Closes #10
  • Loading branch information
bk138 committed Dec 22, 2024
1 parent c0fe8e2 commit 2b05b6c
Show file tree
Hide file tree
Showing 4 changed files with 196 additions and 74 deletions.
13 changes: 11 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,17 @@ find_package(LibVNCServer REQUIRED)
find_package(Threads REQUIRED)
find_library(CARBON_LIBRARY Carbon REQUIRED)
find_library(IOKIT_LIBRARY IOKit REQUIRED)
find_library(IOSURFACE_LIBRARY IOSurface REQUIRED)
find_library(SCREEN_CAPTURE_KIT ScreenCaptureKit REQUIRED)
find_library(CORE_MEDIA CoreMedia)
find_library(CORE_VIDEO CoreVideo)
find_library(FOUNDATION Foundation)

#
# Sources
#
set(SOURCE_FILES
src/ScreenCapturer.h
src/ScreenCapturer.m
src/mac.m
src/icon.icns
)
Expand All @@ -39,7 +44,11 @@ set_source_files_properties(src/icon.icns PROPERTIES MACOSX_PACKAGE_LOCATION Res

add_executable(${PROJECT_NAME} MACOSX_BUNDLE ${SOURCE_FILES})

target_link_libraries(${PROJECT_NAME} LibVNCServer::vncserver ${CMAKE_THREAD_LIBS_INIT} ${CARBON_LIBRARY} ${IOKIT_LIBRARY} ${IOSURFACE_LIBRARY})
target_link_libraries(${PROJECT_NAME} LibVNCServer::vncserver ${CMAKE_THREAD_LIBS_INIT} ${CARBON_LIBRARY} ${IOKIT_LIBRARY}
${SCREEN_CAPTURE_KIT}
${CORE_MEDIA}
${CORE_VIDEO}
${FOUNDATION})

#
# Install, i.e. finalise bundle
Expand Down
18 changes: 18 additions & 0 deletions src/ScreenCapturer.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#import <ScreenCaptureKit/ScreenCaptureKit.h>
#import <CoreGraphics/CoreGraphics.h>
#import <CoreMedia/CoreMedia.h>

NS_ASSUME_NONNULL_BEGIN

@interface ScreenCapturer : NSObject <SCStreamDelegate, SCStreamOutput>

- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID
frameHandler:(nonnull void (^)(CMSampleBufferRef sampleBuffer))frameHandler
errorHandler:(nonnull void (^)(NSError *error))errorHandler;

- (void)startCapture;
- (void)stopCapture;

@end

NS_ASSUME_NONNULL_END
105 changes: 105 additions & 0 deletions src/ScreenCapturer.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#import "ScreenCapturer.h"

@interface ScreenCapturer ()

@property (nonatomic, assign) CGDirectDisplayID displayID;
@property (nonatomic, strong) SCStream *stream;

// handlers
@property (nonatomic, copy, nonnull) void (^frameHandler)(CMSampleBufferRef sampleBuffer);
@property (nonatomic, copy, nonnull) void (^errorHandler)(NSError *error);

@end


@implementation ScreenCapturer

- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID
frameHandler:(void (^)(CMSampleBufferRef))frameHandler
errorHandler:(void (^)(NSError *))errorHandler {
if (self = [super init]) {
_displayID = displayID;
_frameHandler = [frameHandler copy];
_errorHandler = [errorHandler copy];
}
return self;
}

- (void)startCapture {
[SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *error) {
if (error) {
self.errorHandler(error);
return;
}

SCDisplay *display = content.displays[[content.displays indexOfObjectPassingTest:^BOOL(SCDisplay *_Nonnull d, NSUInteger idx, BOOL *_Nonnull stop) {
return d.displayID == self.displayID;
}]];

if (!display) {
NSError *noDisplayError = [NSError errorWithDomain:@"ScreenCapturerErrorDomain"
code:1
userInfo:@{NSLocalizedDescriptionKey : @"Display not available for capture"}];
self.errorHandler(noDisplayError);
return;
}

SCStreamConfiguration *config = [[SCStreamConfiguration alloc] init];
// can later be adjusted for server-side scaling
config.width = display.width;
config.height = display.height;
// set max frame rate to 60 FPS
config.minimumFrameInterval = CMTimeMake(1, 60);
config.pixelFormat = kCVPixelFormatType_32BGRA;

SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:(display) excludingWindows:(@[])];
self.stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:self];

NSError *addOutputError = nil;
[self.stream addStreamOutput:self
type:SCStreamOutputTypeScreen
sampleHandlerQueue:dispatch_queue_create("libvncserver.examples.mac", NULL)
error:&addOutputError];
if (addOutputError) {
self.errorHandler(addOutputError);
return;
}

[self.stream startCaptureWithCompletionHandler:^(NSError * _Nullable startError) {
if (startError) {
self.errorHandler(startError);
}
}];
}];
}

- (void)stopCapture {
[self.stream stopCaptureWithCompletionHandler:^(NSError * _Nullable stopError) {
if (stopError) {
self.errorHandler(stopError);
}
self.stream = nil;
}];
}


/*
SCStreamDelegate methods
*/

- (void) stream:(SCStream *) stream didStopWithError:(NSError *) error {
self.errorHandler(error);
}


/*
SCStreamOutput methods
*/

- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type {
if (type == SCStreamOutputTypeScreen) {
self.frameHandler(sampleBuffer);
}
}

@end
134 changes: 62 additions & 72 deletions src/mac.m
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,17 @@ void PtrAddEvent(buttonMask, x, y, cl)
*/

#include <Carbon/Carbon.h>
#include <ScreenCaptureKit/ScreenCaptureKit.h>
#include <rfb/rfb.h>
#include <rfb/keysym.h>
#include <IOSurface/IOSurface.h>
#include <IOKit/pwr_mgt/IOPMLib.h>
#include <IOKit/pwr_mgt/IOPM.h>
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>

#import "ScreenCapturer.h"

/* The main LibVNCServer screen object */
rfbScreenInfoPtr rfbScreen;
/* Operation modes set by CLI options */
Expand Down Expand Up @@ -560,77 +562,65 @@ rfbBool keyboardInit()
rfbScreen->ptrAddEvent = PtrAddEvent;
rfbScreen->kbdAddEvent = KbdAddEvent;

dispatch_queue_t dispatchQueue = dispatch_queue_create("libvncserver.examples.mac", NULL);
CGDisplayStreamRef stream = CGDisplayStreamCreateWithDispatchQueue(displayID,
CGDisplayPixelsWide(displayID),
CGDisplayPixelsHigh(displayID),
'BGRA',
nil,
dispatchQueue,
^(CGDisplayStreamFrameStatus status,
uint64_t displayTime,
IOSurfaceRef frameSurface,
CGDisplayStreamUpdateRef updateRef) {

if (status == kCGDisplayStreamFrameStatusFrameComplete && frameSurface != NULL) {
rfbClientIteratorPtr iterator;
rfbClientPtr cl;
const CGRect *updatedRects;
size_t updatedRectsCount;
size_t r;

/*
Copy new frame to back buffer.
*/
IOSurfaceLock(frameSurface, kIOSurfaceLockReadOnly, NULL);

memcpy(backBuffer,
IOSurfaceGetBaseAddress(frameSurface),
CGDisplayPixelsWide(displayID) * CGDisplayPixelsHigh(displayID) * 4);

IOSurfaceUnlock(frameSurface, kIOSurfaceLockReadOnly, NULL);

/* Lock out client reads. */
iterator=rfbGetClientIterator(rfbScreen);
while((cl=rfbClientIteratorNext(iterator))) {
LOCK(cl->sendMutex);
}
rfbReleaseClientIterator(iterator);

/* Swap framebuffers. */
if (backBuffer == frameBufferOne) {
backBuffer = frameBufferTwo;
rfbScreen->frameBuffer = frameBufferOne;
} else {
backBuffer = frameBufferOne;
rfbScreen->frameBuffer = frameBufferTwo;
}

/* Mark modified rects in new framebuffer. */
updatedRects = CGDisplayStreamUpdateGetRects(updateRef, kCGDisplayStreamUpdateDirtyRects, &updatedRectsCount);
for(r=0; r<updatedRectsCount; ++r) {
rfbMarkRectAsModified(rfbScreen,
updatedRects[r].origin.x,
updatedRects[r].origin.y,
updatedRects[r].origin.x + updatedRects[r].size.width,
updatedRects[r].origin.y + updatedRects[r].size.height);
}

/* Swapping framebuffers finished, reenable client reads. */
iterator=rfbGetClientIterator(rfbScreen);
while((cl=rfbClientIteratorNext(iterator))) {
UNLOCK(cl->sendMutex);
}
rfbReleaseClientIterator(iterator);
}

});
if(stream) {
CGDisplayStreamStart(stream);
} else {
rfbErr("Could not get screen contents. Check if the program has been given screen recording permissions in 'System Preferences'->'Security & Privacy'->'Privacy'->'Screen Recording'.\n");
return FALSE;
}
ScreenCapturer *capturer = [[ScreenCapturer alloc] initWithDisplay: displayID
frameHandler:^(CMSampleBufferRef sampleBuffer) {
rfbClientIteratorPtr iterator;
rfbClientPtr cl;

/*
Copy new frame to back buffer.
*/
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if(!pixelBuffer)
return;

CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);

memcpy(backBuffer,
CVPixelBufferGetBaseAddress(pixelBuffer),
CGDisplayPixelsWide(displayID) * CGDisplayPixelsHigh(displayID) * 4);

CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);

/* Lock out client reads. */
iterator=rfbGetClientIterator(rfbScreen);
while((cl=rfbClientIteratorNext(iterator))) {
LOCK(cl->sendMutex);
}
rfbReleaseClientIterator(iterator);

/* Swap framebuffers. */
if (backBuffer == frameBufferOne) {
backBuffer = frameBufferTwo;
rfbScreen->frameBuffer = frameBufferOne;
} else {
backBuffer = frameBufferOne;
rfbScreen->frameBuffer = frameBufferTwo;
}

/*
Mark modified rect in new framebuffer.
ScreenCaptureKit does not have something like CGDisplayStreamUpdateGetRects(),
so mark the whole framebuffer.
*/
rfbMarkRectAsModified(rfbScreen, 0, 0, CGDisplayPixelsWide(displayID), CGDisplayPixelsHigh(displayID));

/* Swapping framebuffers finished, reenable client reads. */
iterator=rfbGetClientIterator(rfbScreen);
while((cl=rfbClientIteratorNext(iterator))) {
UNLOCK(cl->sendMutex);
}
rfbReleaseClientIterator(iterator);

} errorHandler:^(NSError *error) {
fprintf(stderr, "Error: %s\n", [error.description UTF8String]);
if(error.code == SCStreamErrorUserDeclined) {
fprintf(stderr, "Could not get screen contents. Check if the program has been given screen recording permissions in 'System Preferences'->'Security & Privacy'->'Privacy'->'Screen Recording'.\n");
}
//TODO handle other errors
exit(EXIT_FAILURE);
}];
[capturer startCapture];

rfbInitServer(rfbScreen);

Expand Down

0 comments on commit 2b05b6c

Please sign in to comment.