oxide_browser/
media_capture.rs

1//! Host-side media capture: camera, microphone, and screen (with permission prompts).
2//!
3//! Guests call [`register_media_capture_functions`] imports from the `oxide` module. Access is
4//! gated per origin through [`crate::permissions`]: the first call returns
5//! [`PERMISSION_PENDING`] while the in-browser prompt is showing, and the guest retries on a
6//! later frame. Native OS prompts (camera / microphone / screen recording) may appear in
7//! addition once the in-browser grant is given.
8
9use std::collections::VecDeque;
10use std::sync::{Arc, Mutex};
11
12use anyhow::Result;
13use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
14use cpal::SampleFormat;
15use nokhwa::pixel_format::RgbFormat;
16use nokhwa::utils::{CameraIndex, RequestedFormat, RequestedFormatType};
17use nokhwa::Camera;
18use wasmtime::{Caller, Linker};
19
20use crate::capabilities::{console_log, write_guest_bytes, ConsoleLevel, HostState};
21use crate::permissions::{check_or_request, PermissionKind, PermissionStatus, PERMISSION_PENDING};
22
23const MIC_RING_CAP: usize = 96_000;
24
25/// Shared capture state for a tab (camera stream, mic ring buffer, counters for pipeline stats).
26#[derive(Default)]
27pub struct MediaCaptureState {
28    camera: Option<Camera>,
29    last_frame_w: u32,
30    last_frame_h: u32,
31    camera_frames: u64,
32    microphone: Option<MicrophoneInput>,
33    screen_w: u32,
34    screen_h: u32,
35    screen_captures: u64,
36}
37
38impl MediaCaptureState {
39    /// Stops any live camera/microphone streams and resets all counters.
40    ///
41    /// Called when the tab navigates to a different origin so a new app can never
42    /// read frames or samples from devices the previous origin opened.
43    pub fn reset(&mut self) {
44        if let Some(mut cam) = self.camera.take() {
45            let _ = cam.stop_stream();
46        }
47        *self = Self::default();
48    }
49}
50
51struct MicrophoneInput {
52    #[allow(dead_code)]
53    stream: cpal::Stream,
54    buffer: Arc<Mutex<VecDeque<f32>>>,
55    sample_rate: u32,
56}
57
58/// Checks the per-origin grant for `kind`.
59///
60/// Returns `None` when granted, or the code the host API should return: `-1` when the user
61/// blocked it (or the app's manifest doesn't declare the permission),
62/// [`PERMISSION_PENDING`] while the in-browser prompt is awaiting a decision.
63fn permission_gate(caller: &Caller<'_, HostState>, kind: PermissionKind) -> Option<i32> {
64    if !crate::manifest::manifest_allows(&caller.data().manifest, kind) {
65        console_log(
66            &caller.data().console,
67            ConsoleLevel::Warn,
68            format!(
69                "[PERMISSIONS] '{}' is not declared in the app manifest — denied",
70                kind.name()
71            ),
72        );
73        return Some(-1);
74    }
75    let origin = caller.data().module_origin.lock().unwrap().clone();
76    match check_or_request(&caller.data().permissions, &origin, kind) {
77        PermissionStatus::Granted => None,
78        PermissionStatus::Denied => Some(-1),
79        PermissionStatus::Pending => Some(PERMISSION_PENDING),
80    }
81}
82
83fn push_mono_f32(data: &[f32], channels: usize, ring: &Arc<Mutex<VecDeque<f32>>>) {
84    let ch = channels.max(1);
85    let frames = data.len() / ch;
86    let mut q = ring.lock().unwrap();
87    for i in 0..frames {
88        let mut sum = 0.0f32;
89        for c in 0..ch {
90            sum += data[i * ch + c];
91        }
92        let m = sum / ch as f32;
93        while q.len() >= MIC_RING_CAP {
94            q.pop_front();
95        }
96        q.push_back(m);
97    }
98}
99
100fn push_mono_i16(data: &[i16], channels: usize, ring: &Arc<Mutex<VecDeque<f32>>>) {
101    let ch = channels.max(1);
102    let frames = data.len() / ch;
103    let mut q = ring.lock().unwrap();
104    for i in 0..frames {
105        let mut sum = 0.0f32;
106        for c in 0..ch {
107            sum += data[i * ch + c] as f32 / 32768.0;
108        }
109        let m = sum / ch as f32;
110        while q.len() >= MIC_RING_CAP {
111            q.pop_front();
112        }
113        q.push_back(m);
114    }
115}
116
117fn push_mono_u16(data: &[u16], channels: usize, ring: &Arc<Mutex<VecDeque<f32>>>) {
118    let ch = channels.max(1);
119    let frames = data.len() / ch;
120    let mut q = ring.lock().unwrap();
121    for i in 0..frames {
122        let mut sum = 0.0f32;
123        for c in 0..ch {
124            sum += (data[i * ch + c] as f32 - 32768.0) / 32768.0;
125        }
126        let m = sum / ch as f32;
127        while q.len() >= MIC_RING_CAP {
128            q.pop_front();
129        }
130        q.push_back(m);
131    }
132}
133
134fn open_microphone(
135    console: &Arc<Mutex<Vec<crate::capabilities::ConsoleEntry>>>,
136) -> Result<MicrophoneInput, i32> {
137    let host = cpal::default_host();
138    let device = host
139        .default_input_device()
140        .ok_or_else(|| log_err(console, -2, "[MIC] No input device".to_string()))?;
141    let supported = match device.default_input_config() {
142        Ok(c) => c,
143        Err(e) => {
144            return Err(log_err(console, -3, format!("[MIC] Config: {e}")));
145        }
146    };
147    let sample_format = supported.sample_format();
148    let config: cpal::StreamConfig = supported.clone().into();
149    let channels = config.channels as usize;
150    let ring = Arc::new(Mutex::new(VecDeque::with_capacity(MIC_RING_CAP)));
151    let ring2 = ring.clone();
152    let console_err = console.clone();
153    let err_fn = move |e| {
154        console_log(
155            &console_err,
156            ConsoleLevel::Warn,
157            format!("[MIC] Stream error: {e}"),
158        );
159    };
160
161    let stream = match sample_format {
162        SampleFormat::F32 => device.build_input_stream(
163            &config,
164            move |data: &[f32], _| push_mono_f32(data, channels, &ring2),
165            err_fn,
166            None,
167        ),
168        SampleFormat::I16 => device.build_input_stream(
169            &config,
170            move |data: &[i16], _| push_mono_i16(data, channels, &ring2),
171            err_fn,
172            None,
173        ),
174        SampleFormat::U16 => device.build_input_stream(
175            &config,
176            move |data: &[u16], _| push_mono_u16(data, channels, &ring2),
177            err_fn,
178            None,
179        ),
180        other => {
181            return Err(log_err(
182                console,
183                -3,
184                format!("[MIC] Unsupported sample format {other:?}"),
185            ));
186        }
187    };
188    let stream = match stream {
189        Ok(s) => s,
190        Err(e) => {
191            return Err(log_err(console, -3, format!("[MIC] Build stream: {e}")));
192        }
193    };
194    if let Err(e) = stream.play() {
195        return Err(log_err(console, -3, format!("[MIC] Play: {e}")));
196    }
197    let sample_rate = supported.sample_rate();
198    Ok(MicrophoneInput {
199        stream,
200        buffer: ring,
201        sample_rate,
202    })
203}
204
205fn log_err(
206    console: &Arc<Mutex<Vec<crate::capabilities::ConsoleEntry>>>,
207    code: i32,
208    msg: String,
209) -> i32 {
210    console_log(console, ConsoleLevel::Warn, msg);
211    code
212}
213
214/// Register `api_camera_*`, `api_microphone_*`, `api_screen_capture`, and `api_media_pipeline_stats`.
215pub fn register_media_capture_functions(linker: &mut Linker<HostState>) -> Result<()> {
216    linker.func_wrap(
217        "oxide",
218        "api_camera_open",
219        |caller: Caller<'_, HostState>| -> i32 {
220            let console = caller.data().console.clone();
221            let st = caller.data().media_capture.clone();
222            if let Some(code) = permission_gate(&caller, PermissionKind::Camera) {
223                return code;
224            }
225            let mut g = st.lock().unwrap();
226            if let Some(mut cam) = g.camera.take() {
227                let _ = cam.stop_stream();
228            }
229            let cams = match nokhwa::query(nokhwa::utils::ApiBackend::Auto) {
230                Ok(c) => c,
231                Err(e) => {
232                    return log_err(&console, -2, format!("[CAMERA] No cameras: {e}"));
233                }
234            };
235            if cams.is_empty() {
236                return log_err(&console, -2, "[CAMERA] No cameras found".to_string());
237            }
238            let req = RequestedFormat::new::<RgbFormat>(RequestedFormatType::HighestResolution(
239                nokhwa::utils::Resolution::new(1280, 720),
240            ));
241            let mut camera = match Camera::new(CameraIndex::Index(0), req) {
242                Ok(c) => c,
243                Err(e) => {
244                    return log_err(&console, -3, format!("[CAMERA] Open failed: {e}"));
245                }
246            };
247            if let Err(e) = camera.open_stream() {
248                return log_err(&console, -3, format!("[CAMERA] Stream: {e}"));
249            }
250            g.camera = Some(camera);
251            0
252        },
253    )?;
254
255    linker.func_wrap(
256        "oxide",
257        "api_camera_close",
258        |caller: Caller<'_, HostState>| {
259            let st = caller.data().media_capture.clone();
260            let mut g = st.lock().unwrap();
261            if let Some(mut cam) = g.camera.take() {
262                let _ = cam.stop_stream();
263            }
264        },
265    )?;
266
267    linker.func_wrap(
268        "oxide",
269        "api_camera_capture_frame",
270        |mut caller: Caller<'_, HostState>, out_ptr: u32, out_cap: u32| -> u32 {
271            let mem = match caller.data().memory {
272                Some(m) => m,
273                None => return 0,
274            };
275            let st = caller.data().media_capture.clone();
276            let mut g = st.lock().unwrap();
277            let cam = match g.camera.as_mut() {
278                Some(c) => c,
279                None => return 0,
280            };
281            let buffer = match cam.frame() {
282                Ok(b) => b,
283                Err(e) => {
284                    console_log(
285                        &caller.data().console,
286                        ConsoleLevel::Warn,
287                        format!("[CAMERA] Frame: {e}"),
288                    );
289                    return 0;
290                }
291            };
292            let img = match buffer.decode_image::<RgbFormat>() {
293                Ok(i) => i,
294                Err(e) => {
295                    console_log(
296                        &caller.data().console,
297                        ConsoleLevel::Warn,
298                        format!("[CAMERA] Decode: {e}"),
299                    );
300                    return 0;
301                }
302            };
303            let w = img.width();
304            let h = img.height();
305            let mut rgba = Vec::with_capacity((w * h * 4) as usize);
306            for px in img.pixels() {
307                let p = px.0;
308                rgba.push(p[0]);
309                rgba.push(p[1]);
310                rgba.push(p[2]);
311                rgba.push(255);
312            }
313            g.last_frame_w = w;
314            g.last_frame_h = h;
315            g.camera_frames = g.camera_frames.saturating_add(1);
316            let write_len = rgba.len().min(out_cap as usize);
317            if write_guest_bytes(&mem, &mut caller, out_ptr, &rgba[..write_len]).is_err() {
318                return 0;
319            }
320            write_len as u32
321        },
322    )?;
323
324    linker.func_wrap(
325        "oxide",
326        "api_camera_frame_dimensions",
327        |caller: Caller<'_, HostState>| -> u64 {
328            let g = caller.data().media_capture.lock().unwrap();
329            ((g.last_frame_w as u64) << 32) | (g.last_frame_h as u64)
330        },
331    )?;
332
333    linker.func_wrap(
334        "oxide",
335        "api_microphone_open",
336        |caller: Caller<'_, HostState>| -> i32 {
337            let console = caller.data().console.clone();
338            let st = caller.data().media_capture.clone();
339            if let Some(code) = permission_gate(&caller, PermissionKind::Microphone) {
340                return code;
341            }
342            let mut g = st.lock().unwrap();
343            g.microphone = None;
344            match open_microphone(&console) {
345                Ok(m) => {
346                    g.microphone = Some(m);
347                    0
348                }
349                Err(code) => code,
350            }
351        },
352    )?;
353
354    linker.func_wrap(
355        "oxide",
356        "api_microphone_close",
357        |caller: Caller<'_, HostState>| {
358            let st = caller.data().media_capture.clone();
359            st.lock().unwrap().microphone = None;
360        },
361    )?;
362
363    linker.func_wrap(
364        "oxide",
365        "api_microphone_sample_rate",
366        |caller: Caller<'_, HostState>| -> u32 {
367            let g = caller.data().media_capture.lock().unwrap();
368            g.microphone.as_ref().map(|m| m.sample_rate).unwrap_or(0)
369        },
370    )?;
371
372    linker.func_wrap(
373        "oxide",
374        "api_microphone_read_samples",
375        |mut caller: Caller<'_, HostState>, out_ptr: u32, max_samples: u32| -> u32 {
376            let mem = match caller.data().memory {
377                Some(m) => m,
378                None => return 0,
379            };
380            let st = caller.data().media_capture.clone();
381            let g = st.lock().unwrap();
382            let mic = match g.microphone.as_ref() {
383                Some(m) => m,
384                None => return 0,
385            };
386            let mut q = mic.buffer.lock().unwrap();
387            let take = (max_samples as usize).min(q.len());
388            let mut chunk = Vec::with_capacity(take * 4);
389            for _ in 0..take {
390                if let Some(s) = q.pop_front() {
391                    chunk.extend_from_slice(&s.to_le_bytes());
392                }
393            }
394            let write_len = chunk.len().min((max_samples as usize).saturating_mul(4));
395            if write_guest_bytes(&mem, &mut caller, out_ptr, &chunk[..write_len]).is_err() {
396                return 0;
397            }
398            (write_len / 4) as u32
399        },
400    )?;
401
402    linker.func_wrap(
403        "oxide",
404        "api_screen_capture",
405        |mut caller: Caller<'_, HostState>, out_ptr: u32, out_cap: u32| -> i32 {
406            let mem = match caller.data().memory {
407                Some(m) => m,
408                None => return -4,
409            };
410            let console = caller.data().console.clone();
411            // The OS may additionally ask for screen-recording permission once granted here.
412            if let Some(code) = permission_gate(&caller, PermissionKind::ScreenCapture) {
413                return code;
414            }
415            let screens = match screenshots::Screen::all() {
416                Ok(s) => s,
417                Err(e) => {
418                    console_log(
419                        &console,
420                        ConsoleLevel::Warn,
421                        format!("[SCREEN] Enumerate: {e}"),
422                    );
423                    return -2;
424                }
425            };
426            let screen = match screens.first() {
427                Some(s) => s,
428                None => {
429                    return log_err(&console, -2, "[SCREEN] No displays".to_string());
430                }
431            };
432            let img = match screen.capture() {
433                Ok(i) => i,
434                Err(e) => {
435                    console_log(
436                        &console,
437                        ConsoleLevel::Warn,
438                        format!("[SCREEN] Capture: {e}"),
439                    );
440                    return -3;
441                }
442            };
443            let w = img.width();
444            let h = img.height();
445            let rgba = img.into_raw();
446            let st = caller.data().media_capture.clone();
447            {
448                let mut g = st.lock().unwrap();
449                g.screen_w = w;
450                g.screen_h = h;
451                g.screen_captures = g.screen_captures.saturating_add(1);
452            }
453            let write_len = rgba.len().min(out_cap as usize);
454            if write_guest_bytes(&mem, &mut caller, out_ptr, &rgba[..write_len]).is_err() {
455                return -4;
456            }
457            write_len as i32
458        },
459    )?;
460
461    linker.func_wrap(
462        "oxide",
463        "api_screen_capture_dimensions",
464        |caller: Caller<'_, HostState>| -> u64 {
465            let g = caller.data().media_capture.lock().unwrap();
466            ((g.screen_w as u64) << 32) | (g.screen_h as u64)
467        },
468    )?;
469
470    linker.func_wrap(
471        "oxide",
472        "api_media_pipeline_stats",
473        |caller: Caller<'_, HostState>| -> u64 {
474            let g = caller.data().media_capture.lock().unwrap();
475            let mic_ring = g
476                .microphone
477                .as_ref()
478                .map(|m| m.buffer.lock().unwrap().len() as u32)
479                .unwrap_or(0);
480            (g.camera_frames << 32) | (mic_ring as u64)
481        },
482    )?;
483
484    Ok(())
485}
oxide_browser/media_capture.rs

oxide_browser/
media_capture.rs
