Initial commit of Edison Voice menu-bar dictation app
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
/// Microphone capture with on-the-fly conversion to whatever format the speech engine wants.
|
||||
///
|
||||
/// The tap runs on a real-time audio thread, so everything it touches lives behind
|
||||
/// `nonisolated(unsafe)` and is only ever mutated from that one thread.
|
||||
public final class AudioCapture: @unchecked Sendable {
|
||||
public init() {}
|
||||
|
||||
private let engine = AVAudioEngine()
|
||||
private nonisolated(unsafe) var converter: AVAudioConverter?
|
||||
private nonisolated(unsafe) var outputFormat: AVAudioFormat?
|
||||
private var isRunning = false
|
||||
|
||||
/// Called on the audio thread with each converted buffer.
|
||||
private nonisolated(unsafe) var onBuffer: (@Sendable (AudioChunk) -> Void)?
|
||||
/// Called on the audio thread with a 0…1 RMS level, for the HUD waveform.
|
||||
private nonisolated(unsafe) var onLevel: (@Sendable (Float) -> Void)?
|
||||
|
||||
public func start(
|
||||
outputFormat: AVAudioFormat,
|
||||
onBuffer: @escaping @Sendable (AudioChunk) -> Void,
|
||||
onLevel: @escaping @Sendable (Float) -> Void
|
||||
) throws {
|
||||
guard !isRunning else { return }
|
||||
|
||||
self.onBuffer = onBuffer
|
||||
self.onLevel = onLevel
|
||||
self.outputFormat = outputFormat
|
||||
|
||||
let input = engine.inputNode
|
||||
let nativeFormat = input.outputFormat(forBus: 0)
|
||||
|
||||
converter = nativeFormat == outputFormat
|
||||
? nil
|
||||
: AVAudioConverter(from: nativeFormat, to: outputFormat)
|
||||
|
||||
input.removeTap(onBus: 0)
|
||||
input.installTap(onBus: 0, bufferSize: 2048, format: nativeFormat) { [weak self] buffer, _ in
|
||||
self?.handle(buffer)
|
||||
}
|
||||
|
||||
engine.prepare()
|
||||
try engine.start()
|
||||
isRunning = true
|
||||
Log.audio.info("capture started — native \(nativeFormat.sampleRate)Hz → engine \(outputFormat.sampleRate)Hz")
|
||||
}
|
||||
|
||||
public func stop() {
|
||||
guard isRunning else { return }
|
||||
engine.inputNode.removeTap(onBus: 0)
|
||||
engine.stop()
|
||||
isRunning = false
|
||||
converter = nil
|
||||
onBuffer = nil
|
||||
onLevel = nil
|
||||
Log.audio.info("capture stopped")
|
||||
}
|
||||
|
||||
// MARK: - Audio thread
|
||||
|
||||
private func handle(_ buffer: AVAudioPCMBuffer) {
|
||||
onLevel?(Self.rms(of: buffer))
|
||||
|
||||
guard let outputFormat else { return }
|
||||
|
||||
// AVAudioEngine reuses the tap's buffer as soon as this returns, so the engine
|
||||
// must never see it directly — copy when no conversion would otherwise allocate.
|
||||
guard let converter else {
|
||||
if let copy = Self.copy(buffer) {
|
||||
onBuffer?(AudioChunk(buffer: copy))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Output frame count scales with the sample-rate ratio; round up so we never clip.
|
||||
let ratio = outputFormat.sampleRate / buffer.format.sampleRate
|
||||
let capacity = AVAudioFrameCount((Double(buffer.frameLength) * ratio).rounded(.up)) + 64
|
||||
guard let converted = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: capacity) else { return }
|
||||
|
||||
// The input block runs synchronously inside `convert`, on this thread.
|
||||
nonisolated(unsafe) let input = buffer
|
||||
let consumed = Latch()
|
||||
var error: NSError?
|
||||
let status = converter.convert(to: converted, error: &error) { _, outStatus in
|
||||
guard !consumed.take() else {
|
||||
outStatus.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
outStatus.pointee = .haveData
|
||||
return input
|
||||
}
|
||||
|
||||
if let error {
|
||||
Log.audio.error("conversion failed: \(error.localizedDescription)")
|
||||
return
|
||||
}
|
||||
guard status != .error, converted.frameLength > 0 else { return }
|
||||
onBuffer?(AudioChunk(buffer: converted))
|
||||
}
|
||||
|
||||
/// Deep-copies a tap buffer into storage we own.
|
||||
private static func copy(_ buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? {
|
||||
guard buffer.frameLength > 0,
|
||||
let copy = AVAudioPCMBuffer(pcmFormat: buffer.format, frameCapacity: buffer.frameLength)
|
||||
else { return nil }
|
||||
|
||||
copy.frameLength = buffer.frameLength
|
||||
let channels = Int(buffer.format.channelCount)
|
||||
let frames = Int(buffer.frameLength)
|
||||
|
||||
if let source = buffer.floatChannelData, let destination = copy.floatChannelData {
|
||||
for channel in 0..<channels {
|
||||
destination[channel].update(from: source[channel], count: frames)
|
||||
}
|
||||
} else if let source = buffer.int16ChannelData, let destination = copy.int16ChannelData {
|
||||
for channel in 0..<channels {
|
||||
destination[channel].update(from: source[channel], count: frames)
|
||||
}
|
||||
} else if let source = buffer.int32ChannelData, let destination = copy.int32ChannelData {
|
||||
for channel in 0..<channels {
|
||||
destination[channel].update(from: source[channel], count: frames)
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
return copy
|
||||
}
|
||||
|
||||
/// One-shot flag. Only touched from the audio thread inside a synchronous call.
|
||||
private final class Latch: @unchecked Sendable {
|
||||
private var fired = false
|
||||
/// - Returns: the value *before* this call, then latches to `true`.
|
||||
func take() -> Bool {
|
||||
defer { fired = true }
|
||||
return fired
|
||||
}
|
||||
}
|
||||
|
||||
private static func rms(of buffer: AVAudioPCMBuffer) -> Float {
|
||||
guard let channel = buffer.floatChannelData?[0] else { return 0 }
|
||||
let count = Int(buffer.frameLength)
|
||||
guard count > 0 else { return 0 }
|
||||
|
||||
var sum: Float = 0
|
||||
for i in 0..<count {
|
||||
let sample = channel[i]
|
||||
sum += sample * sample
|
||||
}
|
||||
let rms = (sum / Float(count)).squareRoot()
|
||||
|
||||
// Map roughly -50…0 dBFS onto 0…1 so quiet speech still moves the meter.
|
||||
let db = 20 * log10(max(rms, 1e-7))
|
||||
return max(0, min(1, (db + 50) / 50))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import AppKit
|
||||
import Carbon.HIToolbox
|
||||
import Foundation
|
||||
|
||||
/// Which modifier key holds the mic open.
|
||||
public enum PushToTalkKey: String, CaseIterable, Sendable {
|
||||
case rightOption
|
||||
case fn
|
||||
case rightCommand
|
||||
|
||||
var keyCode: Int64 {
|
||||
switch self {
|
||||
case .rightOption: Int64(kVK_RightOption) // 61
|
||||
case .fn: Int64(kVK_Function) // 63
|
||||
case .rightCommand: Int64(kVK_RightCommand) // 54
|
||||
}
|
||||
}
|
||||
|
||||
/// Device-*dependent* bit for this specific physical key.
|
||||
///
|
||||
/// `CGEventFlags.maskAlternate` is the union mask — it's set whenever *either* Option
|
||||
/// key is down. Using it means: hold Left ⌥, tap Right ⌥, and the release is invisible
|
||||
/// (the union bit is still set by the left key), so `onRelease` never fires. The mic
|
||||
/// stays open, the HUD stays up, and the next press is swallowed too.
|
||||
///
|
||||
/// These raw values are the NX_DEVICE* masks from IOKit's event system; they carry the
|
||||
/// left/right distinction that the public `CGEventFlags` constants discard.
|
||||
var flag: CGEventFlags {
|
||||
switch self {
|
||||
case .rightOption: CGEventFlags(rawValue: 0x40) // NX_DEVICERALTKEYMASK
|
||||
case .rightCommand: CGEventFlags(rawValue: 0x10) // NX_DEVICERCMDKEYMASK
|
||||
case .fn: .maskSecondaryFn // no left/right variant exists
|
||||
}
|
||||
}
|
||||
|
||||
public var displayName: String {
|
||||
switch self {
|
||||
case .rightOption: "Right ⌥"
|
||||
case .fn: "fn"
|
||||
case .rightCommand: "Right ⌘"
|
||||
}
|
||||
}
|
||||
|
||||
/// Swallowing `fn` would break fn+arrow, fn+delete and the emoji picker, so we let it
|
||||
/// through. Dedicated right-hand modifiers are safe to consume.
|
||||
var shouldConsumeEvent: Bool { self != .fn }
|
||||
}
|
||||
|
||||
/// Watches for a held modifier key using a `CGEventTap`.
|
||||
///
|
||||
/// A tap is required rather than `NSEvent.addGlobalMonitor` because `fn` and left/right
|
||||
/// modifier discrimination don't surface through the higher-level APIs. This needs
|
||||
/// Accessibility permission; without it `CGEvent.tapCreate` returns nil.
|
||||
@MainActor
|
||||
public final class HotkeyMonitor {
|
||||
public init() {}
|
||||
|
||||
private var tap: CFMachPort?
|
||||
private var runLoopSource: CFRunLoopSource?
|
||||
private var isPressed = false
|
||||
|
||||
public var key: PushToTalkKey = .rightOption
|
||||
public var onPress: (() -> Void)?
|
||||
public var onRelease: (() -> Void)?
|
||||
|
||||
/// - Returns: `false` if the tap couldn't be created — almost always missing Accessibility permission.
|
||||
@discardableResult
|
||||
public func start() -> Bool {
|
||||
stop()
|
||||
|
||||
let mask = (1 << CGEventType.flagsChanged.rawValue)
|
||||
let refcon = Unmanaged.passUnretained(self).toOpaque()
|
||||
|
||||
guard let tap = CGEvent.tapCreate(
|
||||
tap: .cgSessionEventTap,
|
||||
place: .headInsertEventTap,
|
||||
options: .defaultTap,
|
||||
eventsOfInterest: CGEventMask(mask),
|
||||
callback: { _, type, event, refcon in
|
||||
guard let refcon else { return Unmanaged.passUnretained(event) }
|
||||
let monitor = Unmanaged<HotkeyMonitor>.fromOpaque(refcon).takeUnretainedValue()
|
||||
|
||||
// CGEvent isn't Sendable, so pull out the plain values before crossing into
|
||||
// actor-isolated code. The tap was added to the main run loop, so this
|
||||
// callback genuinely does run on the main thread.
|
||||
let keyCode = event.getIntegerValueField(.keyboardEventKeycode)
|
||||
let flags = event.flags
|
||||
let consume = MainActor.assumeIsolated {
|
||||
monitor.handle(type: type, keyCode: keyCode, flags: flags)
|
||||
}
|
||||
return consume ? nil : Unmanaged.passUnretained(event)
|
||||
},
|
||||
userInfo: refcon
|
||||
) else {
|
||||
Log.hotkey.error("tapCreate failed — Accessibility permission missing?")
|
||||
return false
|
||||
}
|
||||
|
||||
self.tap = tap
|
||||
let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
|
||||
runLoopSource = source
|
||||
CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes)
|
||||
CGEvent.tapEnable(tap: tap, enable: true)
|
||||
|
||||
Log.hotkey.info("listening for \(self.key.displayName)")
|
||||
return true
|
||||
}
|
||||
|
||||
public func stop() {
|
||||
if let tap {
|
||||
CGEvent.tapEnable(tap: tap, enable: false)
|
||||
}
|
||||
if let runLoopSource {
|
||||
CFRunLoopRemoveSource(CFRunLoopGetCurrent(), runLoopSource, .commonModes)
|
||||
}
|
||||
tap = nil
|
||||
runLoopSource = nil
|
||||
isPressed = false
|
||||
}
|
||||
|
||||
// MARK: - Tap callback
|
||||
|
||||
/// - Returns: `true` if the event should be swallowed rather than passed along.
|
||||
private func handle(type: CGEventType, keyCode: Int64, flags: CGEventFlags) -> Bool {
|
||||
// The system disables a tap that runs too slowly or is interrupted; re-arm it.
|
||||
if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
|
||||
if let tap { CGEvent.tapEnable(tap: tap, enable: true) }
|
||||
return false
|
||||
}
|
||||
|
||||
guard type == .flagsChanged, keyCode == key.keyCode else { return false }
|
||||
|
||||
let nowPressed = flags.contains(key.flag)
|
||||
guard nowPressed != isPressed else { return false }
|
||||
isPressed = nowPressed
|
||||
|
||||
if nowPressed { onPress?() } else { onRelease?() }
|
||||
|
||||
return key.shouldConsumeEvent
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import Foundation
|
||||
|
||||
/// Puts text into whatever field currently has keyboard focus.
|
||||
///
|
||||
/// Two strategies, in order:
|
||||
/// 1. **Accessibility** — set `kAXSelectedTextAttribute` on the focused element. Clean and
|
||||
/// instant, and it leaves the pasteboard untouched.
|
||||
/// 2. **Pasteboard + ⌘V** — works in Electron apps and anything else with a half-hearted
|
||||
/// AX implementation. The previous pasteboard contents are restored afterwards.
|
||||
///
|
||||
/// The catch that makes this non-obvious: **many apps return `.success` from the AX write
|
||||
/// and then do nothing.** Electron (Cursor, VS Code, Slack, Discord), Chrome, and most
|
||||
/// terminal emulators all report `kAXSelectedTextAttribute` as settable, accept the write,
|
||||
/// and silently drop it. So the return value is not evidence of anything — strategy 1 is
|
||||
/// only trusted when the insertion point can be *observed* to have moved.
|
||||
///
|
||||
/// This all works because the HUD is a non-activating panel: focus never leaves the user's
|
||||
/// target app, so "the focused element" is still their text field.
|
||||
@MainActor
|
||||
public enum TextInjector {
|
||||
public static func insert(_ text: String) {
|
||||
guard !text.isEmpty else { return }
|
||||
|
||||
switch insertViaAccessibility(text) {
|
||||
case .inserted:
|
||||
Log.inject.info("inserted via AX (\(text.count) chars)")
|
||||
case .unverified(let reason):
|
||||
Log.inject.info("AX insert not verified (\(reason, privacy: .public)) — pasting")
|
||||
insertViaPasteboard(text)
|
||||
}
|
||||
}
|
||||
|
||||
private enum AXOutcome {
|
||||
case inserted
|
||||
case unverified(String)
|
||||
}
|
||||
|
||||
// MARK: - Strategy 1: Accessibility, verified
|
||||
|
||||
private static func insertViaAccessibility(_ text: String) -> AXOutcome {
|
||||
let systemWide = AXUIElementCreateSystemWide()
|
||||
|
||||
var focused: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(
|
||||
systemWide,
|
||||
kAXFocusedUIElementAttribute as CFString,
|
||||
&focused
|
||||
) == .success, let focused else {
|
||||
return .unverified("no focused element")
|
||||
}
|
||||
|
||||
let element = unsafeDowncast(focused as AnyObject, to: AXUIElement.self)
|
||||
|
||||
var settable: DarwinBoolean = false
|
||||
guard AXUIElementIsAttributeSettable(
|
||||
element,
|
||||
kAXSelectedTextAttribute as CFString,
|
||||
&settable
|
||||
) == .success, settable.boolValue else {
|
||||
return .unverified("selected text not settable")
|
||||
}
|
||||
|
||||
// Without a readable insertion point there's no way to tell a real insert from a
|
||||
// silently-dropped one, so don't gamble — go straight to the fallback.
|
||||
guard let before = selectedRange(of: element) else {
|
||||
return .unverified("no readable selection range")
|
||||
}
|
||||
|
||||
guard AXUIElementSetAttributeValue(
|
||||
element,
|
||||
kAXSelectedTextAttribute as CFString,
|
||||
text as CFString
|
||||
) == .success else {
|
||||
return .unverified("set attribute failed")
|
||||
}
|
||||
|
||||
guard let after = selectedRange(of: element) else {
|
||||
return .unverified("selection range unreadable after write")
|
||||
}
|
||||
|
||||
// Deliberately a *movement* check, not an exact-length check. Falling back after a
|
||||
// write that actually landed would paste the text a second time, and a duplicated
|
||||
// paragraph is far worse than a missing one. Some apps normalize newlines or run
|
||||
// autocorrect, so the caret can legitimately advance by something other than the
|
||||
// UTF-16 count — only a completely unmoved selection proves nothing happened.
|
||||
let unchanged = after.location == before.location && after.length == before.length
|
||||
guard !unchanged else {
|
||||
return .unverified("selection unmoved at \(before.location)")
|
||||
}
|
||||
|
||||
return .inserted
|
||||
}
|
||||
|
||||
private static func selectedRange(of element: AXUIElement) -> CFRange? {
|
||||
var value: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(
|
||||
element,
|
||||
kAXSelectedTextRangeAttribute as CFString,
|
||||
&value
|
||||
) == .success, let value else { return nil }
|
||||
|
||||
let axValue = unsafeDowncast(value as AnyObject, to: AXValue.self)
|
||||
guard AXValueGetType(axValue) == .cfRange else { return nil }
|
||||
|
||||
var range = CFRange()
|
||||
guard AXValueGetValue(axValue, .cfRange, &range) else { return nil }
|
||||
return range
|
||||
}
|
||||
|
||||
// MARK: - Strategy 2: Pasteboard + ⌘V
|
||||
|
||||
private static func insertViaPasteboard(_ text: String) {
|
||||
let pasteboard = NSPasteboard.general
|
||||
let saved = pasteboard.pasteboardItems?.compactMap { item -> [NSPasteboard.PasteboardType: Data] in
|
||||
var copy: [NSPasteboard.PasteboardType: Data] = [:]
|
||||
for type in item.types {
|
||||
if let data = item.data(forType: type) { copy[type] = data }
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
pasteboard.clearContents()
|
||||
pasteboard.setString(text, forType: .string)
|
||||
|
||||
Task { @MainActor in
|
||||
// Give the target app a moment to observe the new pasteboard generation before
|
||||
// ⌘V arrives, or a fast paste can grab the *previous* contents.
|
||||
try? await Task.sleep(for: .milliseconds(40))
|
||||
postCommandV()
|
||||
Log.inject.info("pasted (\(text.count) chars)")
|
||||
|
||||
// The paste is asynchronous in the target app; restore only once it's had time
|
||||
// to read the pasteboard.
|
||||
try? await Task.sleep(for: .milliseconds(500))
|
||||
restore(saved, to: pasteboard)
|
||||
}
|
||||
}
|
||||
|
||||
private static func postCommandV() {
|
||||
guard let source = CGEventSource(stateID: .privateState) else { return }
|
||||
let vKey: CGKeyCode = 9 // kVK_ANSI_V
|
||||
|
||||
guard let down = CGEvent(keyboardEventSource: source, virtualKey: vKey, keyDown: true),
|
||||
let up = CGEvent(keyboardEventSource: source, virtualKey: vKey, keyDown: false)
|
||||
else { return }
|
||||
|
||||
// Set explicitly rather than inheriting live hardware modifier state — the user may
|
||||
// still be resting a finger on something.
|
||||
down.flags = .maskCommand
|
||||
up.flags = .maskCommand
|
||||
|
||||
down.post(tap: .cghidEventTap)
|
||||
up.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
private static func restore(
|
||||
_ saved: [[NSPasteboard.PasteboardType: Data]]?,
|
||||
to pasteboard: NSPasteboard
|
||||
) {
|
||||
guard let saved, !saved.isEmpty else { return }
|
||||
pasteboard.clearContents()
|
||||
let items = saved.map { entry -> NSPasteboardItem in
|
||||
let item = NSPasteboardItem()
|
||||
for (type, data) in entry { item.setData(data, forType: type) }
|
||||
return item
|
||||
}
|
||||
pasteboard.writeObjects(items)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user