Skip to content

Commit

Permalink
Support the OPUS format for Enhanced RTMP.
Browse files Browse the repository at this point in the history
  • Loading branch information
@shogo4405
shogo4405 committed Jan 26, 2025
1 parent 0b1a8b8 commit baf307a
Show file tree
Hide file tree
Showing 9 changed files with 270 additions and 59 deletions.
4 changes: 4 additions & 0 deletions Examples/macOS/CameraIngestViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ final class CameraIngestViewController: NSViewController {
await netStreamSwitcher.setPreference(Preference.default)
let stream = await netStreamSwitcher.stream
if let stream {
var audioSettings = AudioCodecSettings()
audioSettings.format = .opus
await stream.setAudioSettings(audioSettings)

await stream.addOutput(lfView!)
await mixer.addOutput(stream)
}
Expand Down
84 changes: 54 additions & 30 deletions HaishinKit/Sources/Codec/AudioCodec.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import AVFoundation
* - seealso: https://developer.apple.com/library/ios/technotes/tn2236/_index.html
*/
final class AudioCodec {
static let frameCamacity: UInt32 = 1024
static let defaultInputBuffersCursor = 0

/// Specifies the settings for audio codec.
var settings: AudioCodecSettings = .default {
Expand All @@ -32,8 +32,8 @@ final class AudioCodec {
guard inputFormat != oldValue else {
return
}
cursor = 0
inputBuffers.removeAll()
inputBuffersCursor = Self.defaultInputBuffersCursor
outputBuffers.removeAll()
audioConverter = makeAudioConverter()
for _ in 0..<settings.format.inputBufferCounts {
Expand All @@ -43,7 +43,8 @@ final class AudioCodec {
}
}
}
private var cursor: Int = 0
private var audioTime = AudioTime()
private var ringBuffer: AudioRingBuffer?
private var inputBuffers: [AVAudioBuffer] = []
private var continuation: AsyncStream<(AVAudioBuffer, AVAudioTime)>.Continuation? {
didSet {
Expand All @@ -52,6 +53,7 @@ final class AudioCodec {
}
private var outputBuffers: [AVAudioBuffer] = []
private var audioConverter: AVAudioConverter?
private var inputBuffersCursor = AudioCodec.defaultInputBuffersCursor

func append(_ sampleBuffer: CMSampleBuffer) {
guard isRunning else {
Expand Down Expand Up @@ -91,33 +93,49 @@ final class AudioCodec {
return
}
var error: NSError?
let outputBuffer = self.outputBuffer
let outputStatus = audioConverter.convert(to: outputBuffer, error: &error) { _, inputStatus in
switch self.inputBuffer {
case let inputBuffer as AVAudioCompressedBuffer:
inputBuffer.copy(audioBuffer)
case let inputBuffer as AVAudioPCMBuffer:
if !inputBuffer.copy(audioBuffer) {
inputBuffer.muted(true)
if let audioBuffer = audioBuffer as? AVAudioPCMBuffer {
ringBuffer?.append(audioBuffer, when: when)
}
var outputStatus: AVAudioConverterOutputStatus = .endOfStream
repeat {
let outputBuffer = self.outputBuffer
outputStatus = audioConverter.convert(to: outputBuffer, error: &error) { inNumberFrames, inputStatus in
switch self.inputBuffer {
case let inputBuffer as AVAudioCompressedBuffer:
inputBuffer.copy(audioBuffer)
inputStatus.pointee = .haveData
return inputBuffer
case let inputBuffer as AVAudioPCMBuffer:
if inNumberFrames <= (self.ringBuffer?.counts ?? 0) {
_ = self.ringBuffer?.render(inNumberFrames, ioData: inputBuffer.mutableAudioBufferList)
inputBuffer.frameLength = inNumberFrames
inputStatus.pointee = .haveData
self.audioTime.advanced(AVAudioFramePosition(inNumberFrames))
return self.inputBuffer
} else {
inputStatus.pointee = .noDataNow
return nil
}
default:
inputStatus.pointee = .noDataNow
return nil
}
}
switch outputStatus {
case .haveData:
if audioTime.hasAnchor {
continuation?.yield((outputBuffer, audioTime.at))
} else {
continuation?.yield((outputBuffer, when))
}
inputBuffersCursor += 1
if inputBuffersCursor == inputBuffers.count {
inputBuffersCursor = Self.defaultInputBuffersCursor
}
default:
break
releaseOutputBuffer(outputBuffer)
}
inputStatus.pointee = .haveData
return self.inputBuffer
}
switch outputStatus {
case .haveData:
continuation?.yield((outputBuffer, when))
case .error:
break
default:
break
}
cursor += 1
if cursor == inputBuffers.count {
cursor = 0
}
} while(outputStatus == .haveData && settings.format != .pcm)
}

private func makeInputBuffer() -> AVAudioBuffer? {
Expand All @@ -126,8 +144,9 @@ final class AudioCodec {
}
switch inputFormat.formatDescription.mediaSubType {
case .linearPCM:
let buffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: Self.frameCamacity)
buffer?.frameLength = Self.frameCamacity
let frameCapacity = settings.format.makeFramesPerPacket(inputFormat.sampleRate)
let buffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: frameCapacity)
buffer?.frameLength = frameCapacity
return buffer
default:
return AVAudioCompressedBuffer(format: inputFormat, packetCapacity: 1, maximumPacketSize: 1024)
Expand All @@ -145,6 +164,9 @@ final class AudioCodec {
}
let converter = AVAudioConverter(from: inputFormat, to: outputFormat)
settings.apply(converter, oldValue: nil)
if inputFormat.formatDescription.mediaSubType == .linearPCM {
ringBuffer = AudioRingBuffer(inputFormat)
}
return converter
}
}
Expand All @@ -170,7 +192,7 @@ extension AudioCodec: Codec {
}

private var inputBuffer: AVAudioBuffer {
return inputBuffers[cursor]
return inputBuffers[inputBuffersCursor]
}
}

Expand All @@ -180,6 +202,7 @@ extension AudioCodec: Runner {
guard !isRunning else {
return
}
audioTime.reset()
audioConverter?.reset()
isRunning = true
}
Expand All @@ -190,5 +213,6 @@ extension AudioCodec: Runner {
}
isRunning = false
continuation = nil
ringBuffer = nil
}
}
56 changes: 43 additions & 13 deletions HaishinKit/Sources/Codec/AudioCodecSettings.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,20 @@ public struct AudioCodecSettings: Codable, Sendable {
public static let maximumNumberOfChannels: UInt32 = 8

/// The type of the AudioCodec supports format.
enum Format: Codable {
public enum Format: Codable, Sendable {
/// The AAC format.
case aac
/// The OPUS format.
case opus
/// The PCM format.
case pcm

var formatID: AudioFormatID {
switch self {
case .aac:
return kAudioFormatMPEG4AAC
case .opus:
return kAudioFormatOpus
case .pcm:
return kAudioFormatLinearPCM
}
Expand All @@ -30,26 +34,21 @@ public struct AudioCodecSettings: Codable, Sendable {
switch self {
case .aac:
return UInt32(MPEG4ObjectID.AAC_LC.rawValue)
case .opus:
return 0
case .pcm:
return kAudioFormatFlagIsNonInterleaved
| kAudioFormatFlagIsPacked
| kAudioFormatFlagIsFloat
}
}

var framesPerPacket: UInt32 {
switch self {
case .aac:
return 1024
case .pcm:
return 1
}
}

var packetSize: UInt32 {
switch self {
case .aac:
return 1
case .opus:
return 1
case .pcm:
return 1024
}
Expand All @@ -59,6 +58,8 @@ public struct AudioCodecSettings: Codable, Sendable {
switch self {
case .aac:
return 0
case .opus:
return 0
case .pcm:
return 32
}
Expand All @@ -68,6 +69,8 @@ public struct AudioCodecSettings: Codable, Sendable {
switch self {
case .aac:
return 0
case .opus:
return 0
case .pcm:
return (bitsPerChannel / 8)
}
Expand All @@ -77,6 +80,8 @@ public struct AudioCodecSettings: Codable, Sendable {
switch self {
case .aac:
return 0
case .opus:
return 0
case .pcm:
return (bitsPerChannel / 8)
}
Expand All @@ -86,6 +91,8 @@ public struct AudioCodecSettings: Codable, Sendable {
switch self {
case .aac:
return 6
case .opus:
return 6
case .pcm:
return 1
}
Expand All @@ -95,15 +102,32 @@ public struct AudioCodecSettings: Codable, Sendable {
switch self {
case .aac:
return 1
case .opus:
return 1
case .pcm:
return 24
}
}

func makeFramesPerPacket(_ sampleRate: Double) -> UInt32 {
switch self {
case .aac:
return 1024
case .opus:
// https://www.rfc-editor.org/rfc/rfc6716#section-2.1.4
let frameDurationSec = 0.02
return UInt32(sampleRate * frameDurationSec)
case .pcm:
return 1
}
}

func makeAudioBuffer(_ format: AVAudioFormat) -> AVAudioBuffer? {
switch self {
case .aac:
return AVAudioCompressedBuffer(format: format, packetCapacity: 1, maximumPacketSize: 1024 * Int(format.channelCount))
case .opus:
return AVAudioCompressedBuffer(format: format, packetCapacity: 1, maximumPacketSize: 1024 * Int(format.channelCount))
case .pcm:
return AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)
}
Expand All @@ -116,7 +140,7 @@ public struct AudioCodecSettings: Codable, Sendable {
mFormatID: formatID,
mFormatFlags: formatFlags,
mBytesPerPacket: bytesPerPacket,
mFramesPerPacket: framesPerPacket,
mFramesPerPacket: makeFramesPerPacket(format.sampleRate),
mBytesPerFrame: bytesPerFrame,
mChannelsPerFrame: min(
config?.channelCount ?? format.channelCount,
Expand All @@ -142,13 +166,19 @@ public struct AudioCodecSettings: Codable, Sendable {
public var channelMap: [Int]?

/// Specifies the output format.
var format: AudioCodecSettings.Format = .aac
public var format: AudioCodecSettings.Format = .aac

/// Creates a new instance.
public init(bitRate: Int = AudioCodecSettings.defaultBitRate, downmix: Bool = true, channelMap: [Int]? = nil) {
public init(
bitRate: Int = AudioCodecSettings.defaultBitRate,
downmix: Bool = true,
channelMap: [Int]? = nil,
format: AudioCodecSettings.Format = .aac
) {
self.bitRate = bitRate
self.downmix = downmix
self.channelMap = channelMap
self.format = format
}

func apply(_ converter: AVAudioConverter?, oldValue: AudioCodecSettings?) {
Expand Down
30 changes: 30 additions & 0 deletions HaishinKit/Sources/Codec/OpusHeaderPacket.swift
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import CoreMedia
import Foundation

struct OpusHeaderPacket {
static let signature = "OpusHead"

let channels: Int
let sampleRate: Double

var payload: Data {
var data = Data()
data.append(contentsOf: Self.signature.utf8)
data.append(0x01)
data.append(UInt8(channels))
data.append(UInt16(0).data)
data.append(UInt32(sampleRate).data)
data.append(UInt16(0).data)
data.append(0x00)
return data
}

init?(formatDescription: CMFormatDescription?) {
guard
let streamDescription = formatDescription?.audioStreamBasicDescription else {
return nil
}
channels = Int(streamDescription.mChannelsPerFrame)
sampleRate = streamDescription.mSampleRate
}
}
25 changes: 23 additions & 2 deletions HaishinKit/Sources/RTMP/RTMPConnection.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ public actor RTMPConnection: NetworkConnection {
public static let defaultWindowSizeS: Int64 = 250000
/// The supported protocols are rtmp, rtmps, rtmpt and rtmps.
public static let supportedProtocols: Set<String> = ["rtmp", "rtmps"]
/// The supported fourCcList are hvc1.
public static let supportedFourCcList = ["hvc1"]
/// The supported fourCcList.
public static let supportedFourCcList = [RTMPVideoFourCC.hevc.description, RTMPAudioFourCC.opus.description]
/// The default RTMP port is 1935.
public static let defaultPort: Int = 1935
/// The default RTMPS port is 443.
Expand All @@ -51,6 +51,14 @@ public actor RTMPConnection: NetworkConnection {
/// The default an rtmp request time out value (ms).
public static let defaultRequestTimeout: UInt64 = 3000

static let videoFourCcInfoMap: AMFObject = [
RTMPVideoFourCC.hevc.description: FourCcInfoMask.canDecode.rawValue | FourCcInfoMask.canEncode.rawValue
]

static let audioFourCcInfoMap: AMFObject = [
RTMPAudioFourCC.opus.description: FourCcInfoMask.canEncode.rawValue
]

private static let connectTransactionId = 1

/**
Expand Down Expand Up @@ -134,6 +142,19 @@ public actor RTMPConnection: NetworkConnection {
case clientSeek = 1
}

enum FourCcInfoMask: Int {
case canDecode = 0x01
case canEncode = 0x02
case canForward = 0x04
}

enum CapsEx: Int {
case recoonect = 0x01
case multitrack = 0x02
case modEx = 0x04
case timestampNanoOffset = 0x08
}

/// The URL of .swf.
public let swfUrl: String?
/// The URL of an HTTP referer.
Expand Down
Loading

0 comments on commit baf307a

Please sign in to comment.