From 8fb2d02315a28abc5608b433b2916611dd28694d Mon Sep 17 00:00:00 2001 From: shogo4405 Date: Sat, 6 Apr 2024 17:44:22 +0900 Subject: [PATCH 1/2] Remove IOScreenCaptureUnit.swift --- HaishinKit.xcodeproj/project.pbxproj | 8 -- Sources/IO/IOScreenCaptureUnit.swift | 16 --- Sources/IO/IOStream.swift | 35 ----- Sources/IO/IOUIScreenCaptureUnit.swift | 175 ------------------------- 4 files changed, 234 deletions(-) delete mode 100644 Sources/IO/IOScreenCaptureUnit.swift delete mode 100644 Sources/IO/IOUIScreenCaptureUnit.swift diff --git a/HaishinKit.xcodeproj/project.pbxproj b/HaishinKit.xcodeproj/project.pbxproj index 6bde79987..1588630a1 100644 --- a/HaishinKit.xcodeproj/project.pbxproj +++ b/HaishinKit.xcodeproj/project.pbxproj @@ -152,7 +152,6 @@ BC1DC50E2A039E1900E928ED /* FLVVideoPacketType.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC50D2A039E1900E928ED /* FLVVideoPacketType.swift */; }; BC1DC5122A04E46E00E928ED /* HEVCDecoderConfigurationRecordTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC5112A04E46E00E928ED /* HEVCDecoderConfigurationRecordTests.swift */; }; BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC1DC5132A05428800E928ED /* HEVCNALUnit.swift */; }; - BC20DF38250377A3007BC608 /* IOUIScreenCaptureUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = 299B131C1D35272D00A1E8F5 /* IOUIScreenCaptureUnit.swift */; }; BC22EEEE2AAF50F200E3406D /* Codec.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC22EEED2AAF50F200E3406D /* Codec.swift */; }; BC22EEF22AAF5D6300E3406D /* AVAudioPCMBuffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC22EEF12AAF5D6300E3406D /* AVAudioPCMBuffer+Extension.swift */; }; BC2828AD2AA3225100741013 /* AVCaptureDevice.Format+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC2828AC2AA3225100741013 /* AVCaptureDevice.Format+Extension.swift */; }; @@ -228,7 +227,6 @@ BC959F1229717EDB0067BA97 /* PreferenceViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC959F1129717EDB0067BA97 /* PreferenceViewController.swift */; }; BC9CFA9323BDE8B700917EEF /* IOStreamView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9CFA9223BDE8B700917EEF /* IOStreamView.swift */; }; BC9F9C7826F8C16600B01ED0 /* Choreographer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */; }; - BCA2252C293CC5B600DD7CB2 /* IOScreenCaptureUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA2252B293CC5B600DD7CB2 /* IOScreenCaptureUnit.swift */; }; BCA7C24F2A91AA0500882D85 /* IORecorderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA7C24E2A91AA0500882D85 /* IORecorderTests.swift */; }; BCAD0C18263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v.m3u8 in Resources */ = {isa = PBXBuildFile; fileRef = BCAD0C16263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v.m3u8 */; }; BCAD0C19263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v in Resources */ = {isa = PBXBuildFile; fileRef = BCAD0C17263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v */; }; @@ -508,7 +506,6 @@ 298BCF321DD4C44A007FF86A /* AnyUtil.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnyUtil.swift; sourceTree = ""; }; 2997BDD31D50D31B000AF900 /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 2999C3742071138F00892E55 /* MTHKView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MTHKView.swift; sourceTree = ""; }; - 299B131C1D35272D00A1E8F5 /* IOUIScreenCaptureUnit.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = IOUIScreenCaptureUnit.swift; sourceTree = ""; }; 299F7E3A1CD71A97001E7272 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 299F7E3B1CD71A97001E7272 /* HaishinKit.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HaishinKit.h; sourceTree = ""; }; 29A39C881D85BF30007C27E9 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -650,7 +647,6 @@ BC959F1129717EDB0067BA97 /* PreferenceViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PreferenceViewController.swift; sourceTree = ""; }; BC9CFA9223BDE8B700917EEF /* IOStreamView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOStreamView.swift; sourceTree = ""; }; BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Choreographer.swift; sourceTree = ""; }; - BCA2252B293CC5B600DD7CB2 /* IOScreenCaptureUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOScreenCaptureUnit.swift; sourceTree = ""; }; BCA7C24E2A91AA0500882D85 /* IORecorderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IORecorderTests.swift; sourceTree = ""; }; BCAD0C16263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v.m3u8 */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "SampleVideo_360x240_5mb@m4v.m3u8"; sourceTree = ""; }; BCAD0C17263ED67F00ADFB80 /* SampleVideo_360x240_5mb@m4v */ = {isa = PBXFileReference; lastKnownFileType = folder; path = "SampleVideo_360x240_5mb@m4v"; sourceTree = ""; }; @@ -1097,13 +1093,11 @@ BC0F1FD92ACC4CC100C326FF /* IOCaptureVideoPreview.swift */, 29B8768B1CD70AFE00FC07DA /* IOMixer.swift */, BC4078C32AD5CC7E00BBB4FA /* IOMuxer.swift */, - BCA2252B293CC5B600DD7CB2 /* IOScreenCaptureUnit.swift */, 29AF3FCE1D7C744C00E41212 /* IOStream.swift */, BC6692F22AC2F717009EC058 /* IOStreamBitRateStrategyConvertible.swift */, 2976A47D1D48C5C700B53EF2 /* IOStreamRecorder.swift */, BC9CFA9223BDE8B700917EEF /* IOStreamView.swift */, BCC4F4142AD6FC1100954EF5 /* IOTellyUnit.swift */, - 299B131C1D35272D00A1E8F5 /* IOUIScreenCaptureUnit.swift */, BC570B4728E9ACC10098A12C /* IOUnit.swift */, BC3802112AB5E770001AE399 /* IOVideoCaptureUnit.swift */, BC3483692AC56F3A002926F1 /* IOVideoMixer.swift */, @@ -1787,7 +1781,6 @@ 2958910E1EEB8D3C00CE51E1 /* FLVVideoCodec.swift in Sources */, B3D687822B80302B00E6A28E /* IOAudioMixer.swift in Sources */, BC1DC5142A05428800E928ED /* HEVCNALUnit.swift in Sources */, - BC20DF38250377A3007BC608 /* IOUIScreenCaptureUnit.swift in Sources */, 29B876AF1CD70B2800FC07DA /* RTMPChunk.swift in Sources */, 29B876841CD70AE800FC07DA /* AVCDecoderConfigurationRecord.swift in Sources */, 296242621D8DB86500C451A3 /* TSWriter.swift in Sources */, @@ -1826,7 +1819,6 @@ BC22EEF22AAF5D6300E3406D /* AVAudioPCMBuffer+Extension.swift in Sources */, BCCBCE9729A90D880095B51C /* AVCNALUnit.swift in Sources */, 29B876BD1CD70B3900FC07DA /* CRC32.swift in Sources */, - BCA2252C293CC5B600DD7CB2 /* IOScreenCaptureUnit.swift in Sources */, BC4914A628DDD367009E2DF6 /* VTSessionOption.swift in Sources */, BC0F1FDA2ACC4CC100C326FF /* IOCaptureVideoPreview.swift in Sources */, BC4914B228DDFE31009E2DF6 /* VTSessionOptionKey.swift in Sources */, diff --git a/Sources/IO/IOScreenCaptureUnit.swift b/Sources/IO/IOScreenCaptureUnit.swift deleted file mode 100644 index 5828d9a72..000000000 --- a/Sources/IO/IOScreenCaptureUnit.swift +++ /dev/null @@ -1,16 +0,0 @@ -import CoreMedia -import Foundation - -/// The interface a capture session uses to inform its delegate. -public protocol IOScreenCaptureUnitDelegate: AnyObject { - /// Tells the receiver to output a pixel buffer. - func session(_ session: any IOScreenCaptureUnit, didOutput pixelBuffer: CVPixelBuffer, presentationTime: CMTime) -} - -/// The interface that provides methods to screen capture. -public protocol IOScreenCaptureUnit: Running { - /// Specifies the CVPixelBufferPool's attributes. - var attributes: [NSString: NSObject] { get } - /// Specifies the delegate. - var delegate: (any IOScreenCaptureUnitDelegate)? { get set } -} diff --git a/Sources/IO/IOStream.swift b/Sources/IO/IOStream.swift index a6097a454..48a54e955 100644 --- a/Sources/IO/IOStream.swift +++ b/Sources/IO/IOStream.swift @@ -520,38 +520,3 @@ extension IOStream: IOTellyUnitDelegate { } } -extension IOStream: IOScreenCaptureUnitDelegate { - // MARK: IOScreenCaptureUnitDelegate - public func session(_ session: any IOScreenCaptureUnit, didOutput pixelBuffer: CVPixelBuffer, presentationTime: CMTime) { - var timingInfo = CMSampleTimingInfo( - duration: .invalid, - presentationTimeStamp: presentationTime, - decodeTimeStamp: .invalid - ) - var videoFormatDescription: CMVideoFormatDescription? - var status = CMVideoFormatDescriptionCreateForImageBuffer( - allocator: kCFAllocatorDefault, - imageBuffer: pixelBuffer, - formatDescriptionOut: &videoFormatDescription - ) - guard status == noErr else { - return - } - var sampleBuffer: CMSampleBuffer? - status = CMSampleBufferCreateForImageBuffer( - allocator: kCFAllocatorDefault, - imageBuffer: pixelBuffer, - dataReady: true, - makeDataReadyCallback: nil, - refcon: nil, - formatDescription: videoFormatDescription!, - sampleTiming: &timingInfo, - sampleBufferOut: &sampleBuffer - ) - guard let sampleBuffer, status == noErr else { - return - } - append(sampleBuffer) - } -} - diff --git a/Sources/IO/IOUIScreenCaptureUnit.swift b/Sources/IO/IOUIScreenCaptureUnit.swift deleted file mode 100644 index 8204e302d..000000000 --- a/Sources/IO/IOUIScreenCaptureUnit.swift +++ /dev/null @@ -1,175 +0,0 @@ -#if os(iOS) || os(tvOS) - -import AVFoundation -import CoreImage -import UIKit - -private extension CGRect { - init(size: CGSize) { - self.init(origin: .zero, size: size) - } -} - -// MARK: - -/// The IOUIScreenCaptureUnit class captures the UIView. -public class IOUIScreenCaptureUnit: NSObject, IOScreenCaptureUnit { - static let defaultPreferredFramesPerSecond: Int = 30 - static let defaultAttributes: [NSString: NSObject] = [ - kCVPixelBufferPixelFormatTypeKey: NSNumber(value: kCVPixelFormatType_32BGRA), - kCVPixelBufferCGBitmapContextCompatibilityKey: true as NSObject - ] - - /// Specifies the boolean value that indicates whether the snapshot image downsize or not. - public var enabledScale = false - /// Specifies the boolean value that indicates whether the snapshot should be rendered after recent changes have been incorporated. - public var afterScreenUpdates = false - /// Specifies the number of shaphot that must pass before the display link notifies the target again. - public var preferredFramesPerSecond: Int = IOUIScreenCaptureUnit.defaultPreferredFramesPerSecond - /// Specifies the CVPixelBufferPool's attrivutes. - public var attributes: [NSString: NSObject] { - var attributes: [NSString: NSObject] = IOUIScreenCaptureUnit.defaultAttributes - attributes[kCVPixelBufferWidthKey] = NSNumber(value: Float(size.width * scale)) - attributes[kCVPixelBufferHeightKey] = NSNumber(value: Float(size.height * scale)) - attributes[kCVPixelBufferBytesPerRowAlignmentKey] = NSNumber(value: Float(size.width * scale * 4)) - return attributes - } - public weak var delegate: (any IOScreenCaptureUnitDelegate)? - public private(set) var isRunning: Atomic = .init(false) - - private var shared: UIApplication? - private var viewToCapture: UIView? - private var context = CIContext(options: [.useSoftwareRenderer: NSNumber(value: false)]) - private let semaphore = DispatchSemaphore(value: 1) - private let lockQueue = DispatchQueue( - label: "com.haishinkit.HaishinKit.ScreenCaptureSession.lock", qos: .userInteractive, attributes: [] - ) - private var colorSpace: CGColorSpace! - private var displayLink: CADisplayLink! - - private var size: CGSize = .zero { - didSet { - guard size != oldValue else { - return - } - pixelBufferPool = nil - } - } - private var scale: CGFloat { - enabledScale ? UIScreen.main.scale : 1.0 - } - - private var _pixelBufferPool: CVPixelBufferPool? - private var pixelBufferPool: CVPixelBufferPool! { - get { - if _pixelBufferPool == nil { - var pixelBufferPool: CVPixelBufferPool? - CVPixelBufferPoolCreate(nil, nil, attributes as CFDictionary?, &pixelBufferPool) - _pixelBufferPool = pixelBufferPool - } - return _pixelBufferPool! - } - set { - _pixelBufferPool = newValue - } - } - - /// Creates an IOUIScreenCaptureUnit object to capture UIApplication. - public init(shared: UIApplication) { - self.shared = shared - size = UIScreen.main.bounds.size - super.init() - } - - /// Create an IOUIScreenCaptureUnit object to capture UIView. - public init(viewToCapture: UIView) { - self.viewToCapture = viewToCapture - size = viewToCapture.bounds.size - afterScreenUpdates = true - super.init() - } - - @objc - public func onScreen(_ displayLink: CADisplayLink) { - guard semaphore.wait(timeout: .now()) == .success else { - return - } - - if self.shared != nil { - size = UIScreen.main.bounds.size - } - if let viewToCapture = self.viewToCapture { - size = viewToCapture.bounds.size - } - - lockQueue.async { - autoreleasepool { - self.onScreenProcess(displayLink) - } - self.semaphore.signal() - } - } - - func onScreenProcess(_ displayLink: CADisplayLink) { - var pixelBuffer: CVPixelBuffer? - - CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) - CVPixelBufferLockBaseAddress(pixelBuffer!, []) - UIGraphicsBeginImageContextWithOptions(size, false, scale) - let cgctx: CGContext = UIGraphicsGetCurrentContext()! - DispatchQueue.main.sync { - UIGraphicsPushContext(cgctx) - if let shared: UIApplication = shared { - for window: UIWindow in shared.windows { - window.drawHierarchy( - in: CGRect(x: 0, y: 0, width: self.size.width, height: self.size.height), - afterScreenUpdates: self.afterScreenUpdates - ) - } - } - if let viewToCapture: UIView = viewToCapture { - viewToCapture.drawHierarchy( - in: CGRect(x: 0, y: 0, width: self.size.width, height: self.size.height), - afterScreenUpdates: self.afterScreenUpdates - ) - } - UIGraphicsPopContext() - } - let image: UIImage = UIGraphicsGetImageFromCurrentImageContext()! - UIGraphicsEndImageContext() - context.render(CIImage(cgImage: image.cgImage!), to: pixelBuffer!) - delegate?.session(self, didOutput: pixelBuffer!, presentationTime: CMTimeMakeWithSeconds(displayLink.timestamp, preferredTimescale: 1000)) - CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) - } -} - -extension IOUIScreenCaptureUnit: Running { - // MARK: Running - public func startRunning() { - lockQueue.sync { - guard !self.isRunning.value else { - return - } - self.isRunning.mutate { $0 = true } - self.pixelBufferPool = nil - self.colorSpace = CGColorSpaceCreateDeviceRGB() - self.displayLink = CADisplayLink(target: self, selector: #selector(onScreen)) - self.displayLink.preferredFramesPerSecond = self.preferredFramesPerSecond - self.displayLink.add(to: .main, forMode: .common) - } - } - - public func stopRunning() { - lockQueue.sync { - guard self.isRunning.value else { - return - } - self.displayLink.remove(from: .main, forMode: .common) - self.displayLink.invalidate() - self.colorSpace = nil - self.displayLink = nil - self.isRunning.mutate { $0 = false } - } - } -} - -#endif From 44c33358df73522cb39898e3a79c47f3e1e69a6c Mon Sep 17 00:00:00 2001 From: shogo4405 Date: Sat, 6 Apr 2024 17:50:03 +0900 Subject: [PATCH 2/2] Remove attachScreen feature on macOS. --- Sources/IO/IOStream.swift | 10 ---------- Sources/IO/IOVideoCaptureUnit.swift | 13 ------------- Sources/IO/IOVideoUnit.swift | 12 ------------ 3 files changed, 35 deletions(-) diff --git a/Sources/IO/IOStream.swift b/Sources/IO/IOStream.swift index 48a54e955..bba5fe8b2 100644 --- a/Sources/IO/IOStream.swift +++ b/Sources/IO/IOStream.swift @@ -349,15 +349,6 @@ open class IOStream: NSObject { } #endif - #if os(macOS) - /// Attaches the screen input object. - public func attachScreen(_ input: AVCaptureScreenInput?, channel: UInt8 = 0) { - lockQueue.async { - self.mixer.videoIO.attachScreen(input, channel: channel) - } - } - #endif - /// Append a CMSampleBuffer. /// - Warning: This method can't use attachCamera or attachAudio method at the same time. public func append(_ sampleBuffer: CMSampleBuffer, channel: UInt8 = 0) { @@ -519,4 +510,3 @@ extension IOStream: IOTellyUnitDelegate { }) } } - diff --git a/Sources/IO/IOVideoCaptureUnit.swift b/Sources/IO/IOVideoCaptureUnit.swift index 842abc185..d24923e31 100644 --- a/Sources/IO/IOVideoCaptureUnit.swift +++ b/Sources/IO/IOVideoCaptureUnit.swift @@ -129,19 +129,6 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { setSampleBufferDelegate(videoUnit) } - #if os(macOS) - func attachScreen(_ screen: AVCaptureScreenInput?, videoUnit: IOVideoUnit) { - setSampleBufferDelegate(nil) - videoUnit.mixer?.session.detachCapture(self) - device = nil - input = screen - output = AVCaptureVideoDataOutput() - connection = nil - videoUnit.mixer?.session.attachCapture(self) - setSampleBufferDelegate(videoUnit) - } - #endif - func setFrameRate(_ frameRate: Float64) { guard let device else { return diff --git a/Sources/IO/IOVideoUnit.swift b/Sources/IO/IOVideoUnit.swift index 9b10e7819..108ff4677 100644 --- a/Sources/IO/IOVideoUnit.swift +++ b/Sources/IO/IOVideoUnit.swift @@ -241,18 +241,6 @@ final class IOVideoUnit: NSObject, IOUnit { } } } - - #if os(macOS) - func attachScreen(_ input: AVCaptureScreenInput?, channel: UInt8) { - mixer?.session.configuration { _ in - let capture = capture(for: channel) - for capture in captures.values where capture.input == input { - capture.attachScreen(nil, videoUnit: self) - } - capture?.attachScreen(input, videoUnit: self) - } - } - #endif } extension IOVideoUnit: Running {