Send embed bytes over IPC without JSON

Embeds went over IPC as JSON number arrays, so a 9MB image turned into
~36MB of JSON. On upload Tauri parses that on the main thread, which
beachballed the app while dropping a large screenshot.
This commit is contained in:
Rūdolfs Ošiņš
2026-10-06 15:53:56 +02:00
parent cd8a32a5e7
commit b2bbf6f5ad
13 changed files with 185 additions and 65 deletions
+16 -5
View File
@@ -19,8 +19,10 @@ pub async fn get_embed(
rid: identity::RepoId,
name: Option<String>,
oid: git::Oid,
) -> Result<types::cobs::EmbedWithMimeType, Error> {
ctx.get_embed(rid, name, oid)
) -> Result<tauri::ipc::Response, Error> {
let embed = ctx.get_embed(rid, name, oid)?;
Ok(tauri::ipc::Response::new(embed.into_bytes()))
}
#[tauri::command]
@@ -35,10 +37,19 @@ pub async fn save_embed_by_path(
#[tauri::command]
pub async fn save_embed_by_bytes(
ctx: tauri::State<'_, AppState>,
rid: identity::RepoId,
name: String,
bytes: Vec<u8>,
request: tauri::ipc::Request<'_>,
) -> Result<git::Oid, Error> {
let tauri::ipc::InvokeBody::Raw(body) = request.body() else {
return Err(Error::SaveEmbedError);
};
let rid = request
.headers()
.get("rid")
.and_then(|rid| rid.to_str().ok())
.and_then(|rid| rid.parse().ok())
.ok_or(Error::SaveEmbedError)?;
let (name, bytes) = types::cobs::split_embed_upload(body).ok_or(Error::SaveEmbedError)?;
ctx.save_embed_by_bytes(rid, name, bytes)
}
@@ -1,3 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Embed = { content: Array<number>; mimeType: string | null };
@@ -1,6 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type EmbedWithMimeType = {
content: Array<number>;
mimeType: string | null;
};
+57 -4
View File
@@ -101,15 +101,29 @@ pub struct Operation<A> {
pub timestamp: cob::Timestamp,
}
#[derive(Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
#[ts(export_to = "cob/")]
pub struct EmbedWithMimeType {
pub content: Vec<u8>,
pub mime_type: Option<String>,
}
impl EmbedWithMimeType {
pub fn into_bytes(self) -> Vec<u8> {
let mime_type = self.mime_type.unwrap_or_default();
let mut bytes = Vec::with_capacity(mime_type.len() + 1 + self.content.len());
bytes.extend_from_slice(mime_type.as_bytes());
bytes.push(0);
bytes.extend_from_slice(&self.content);
bytes
}
}
pub fn split_embed_upload(body: &[u8]) -> Option<(&str, &[u8])> {
let end = body.iter().position(|&byte| byte == 0)?;
let name = std::str::from_utf8(&body[..end]).ok()?;
Some((name, &body[end + 1..]))
}
#[derive(TS, Serialize)]
#[doc = "A type alias for the TS type `never`."]
#[ts(export)]
@@ -197,3 +211,42 @@ pub mod query {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn embed_into_bytes() {
let embed = EmbedWithMimeType {
content: vec![1, 0, 2],
mime_type: Some("image/png".to_string()),
};
assert_eq!(embed.into_bytes(), b"image/png\0\x01\0\x02");
let embed = EmbedWithMimeType {
content: vec![1],
mime_type: None,
};
assert_eq!(embed.into_bytes(), b"\0\x01");
}
#[test]
fn split_embed_upload_at_first_nul() {
let body = "a\u{202f}b.png\0"
.bytes()
.chain([1, 0, 2])
.collect::<Vec<_>>();
assert_eq!(
split_embed_upload(&body),
Some(("a\u{202f}b.png", &[1, 0, 2][..]))
);
assert_eq!(split_embed_upload(b"name\0"), Some(("name", &[][..])));
}
#[test]
fn split_embed_upload_rejects_malformed_bodies() {
assert_eq!(split_embed_upload(b"no separator"), None);
assert_eq!(split_embed_upload(b"\xff\0content"), None);
}
}
+3 -3
View File
@@ -78,8 +78,8 @@ pub trait Thread: Profile {
fn save_embed_by_bytes(
&self,
rid: identity::RepoId,
name: String,
bytes: Vec<u8>,
name: &str,
bytes: &[u8],
) -> Result<git::Oid, Error> {
let file_size = bytes.len();
if file_size > MAX_EMBED_SIZE {
@@ -87,7 +87,7 @@ pub trait Thread: Profile {
}
let profile = self.profile();
let repo = profile.storage.repository(rid)?;
let embed = radicle::cob::Embed::<git::Oid>::store(&name, &bytes, &repo.backend)?;
let embed = radicle::cob::Embed::<git::Oid>::store(name, bytes, &repo.backend)?;
Ok(embed.oid())
}
+16 -11
View File
@@ -3,12 +3,14 @@ use std::path::PathBuf;
use std::sync::Arc;
use axum::Router;
use axum::body::Bytes;
use axum::extract::{FromRequestParts, Request, State};
use axum::middleware::{self, Next};
use axum::response::{IntoResponse, Json, Response};
use axum::routing::post;
use hyper::HeaderMap;
use hyper::Method;
use hyper::header::{CONTENT_TYPE, HeaderValue};
use hyper::header::{CONTENT_TYPE, HeaderName, HeaderValue};
use hyper::http::request::Parts;
use serde::{Deserialize, Serialize};
use tokio::sync::{Mutex, RwLock};
@@ -205,7 +207,7 @@ pub fn router(shared: Shared) -> Router {
ALLOWED_ORIGINS.map(HeaderValue::from_static),
))
.allow_methods([Method::POST, Method::GET])
.allow_headers([CONTENT_TYPE]),
.allow_headers([CONTENT_TYPE, HeaderName::from_static("rid")]),
)
.with_state(shared)
}
@@ -886,7 +888,10 @@ async fn get_embeds_handler(
) -> impl IntoResponse {
let embed = ctx.get_embed(rid, name, oid)?;
Ok::<_, Error>(Json(embed))
Ok::<_, Error>((
[(CONTENT_TYPE, "application/octet-stream")],
embed.into_bytes(),
))
}
#[derive(Deserialize)]
@@ -904,17 +909,17 @@ async fn save_embed_by_path_handler(
Ok::<_, Error>(Json(oid))
}
#[derive(Deserialize)]
struct SaveEmbedByBytesBody {
pub rid: identity::RepoId,
pub name: String,
pub bytes: Vec<u8>,
}
async fn save_embed_by_bytes_handler(
Ctx(ctx): Ctx,
Json(SaveEmbedByBytesBody { rid, name, bytes }): Json<SaveEmbedByBytesBody>,
headers: HeaderMap,
body: Bytes,
) -> impl IntoResponse {
let rid = headers
.get("rid")
.and_then(|rid| rid.to_str().ok())
.and_then(|rid| rid.parse().ok())
.ok_or(Error::SaveEmbedError)?;
let (name, bytes) = types::cobs::split_embed_upload(&body).ok_or(Error::SaveEmbedError)?;
let oid = ctx.save_embed_by_bytes(rid, name, bytes)?;
Ok::<_, Error>(Json(oid))
+6 -6
View File
@@ -7,7 +7,7 @@
import debounce from "lodash/debounce";
import { SvelteMap } from "svelte/reactivity";
import { basename } from "@app/lib/embeds";
import { basename, encodeEmbedUpload } from "@app/lib/embeds";
import { hints } from "@app/lib/hints";
import { invoke } from "@app/lib/invoke";
import { matchesShortcut } from "@app/lib/shortcuts.svelte";
@@ -180,11 +180,11 @@
if (file.size > MAX_EMBED_SIZE) {
throw new Error(`${file.name} exceeds the embed size limit`);
}
return invoke<string>("save_embed_by_bytes", {
rid,
name: file.name,
bytes: new Uint8Array(await file.arrayBuffer()),
});
return invoke<string>(
"save_embed_by_bytes",
encodeEmbedUpload(file.name, new Uint8Array(await file.arrayBuffer())),
{ headers: { rid } },
);
}
function hasFiles(event: DragEvent): boolean {
+7 -9
View File
@@ -1,10 +1,8 @@
<script lang="ts">
import type { Embed } from "@bindings/cob/Embed";
import { toDom } from "hast-util-to-dom";
import { mount, tick, unmount } from "svelte";
import { embedPreviewKind, retinaWidth } from "@app/lib/embeds";
import { decodeEmbed, embedPreviewKind, retinaWidth } from "@app/lib/embeds";
import { parseFrontmatter } from "@app/lib/frontmatter";
import { invoke } from "@app/lib/invoke";
import {
@@ -71,7 +69,7 @@
});
function createEmbedPreview(
mimeType: string | null,
mimeType: string | undefined,
url: string,
content: Uint8Array,
): HTMLElement | undefined {
@@ -302,20 +300,20 @@
if (cached) {
placeEmbedPreview(e, cached.element);
} else {
void invoke<Embed>("get_embed", {
void invoke<ArrayBuffer>("get_embed", {
rid,
name: e.innerText,
oid: href,
})
.then(({ mimeType, content }) => {
.then(buffer => {
if (destroyed) {
return;
}
let preview = embedPreviews.get(href);
if (!preview) {
const bytes = Buffer.from(content);
const url = URL.createObjectURL(new Blob([bytes]));
const element = createEmbedPreview(mimeType, url, bytes);
const { mimeType, content } = decodeEmbed(buffer);
const url = URL.createObjectURL(new Blob([content]));
const element = createEmbedPreview(mimeType, url, content);
if (!element) {
URL.revokeObjectURL(url);
console.warn(
+22 -1
View File
@@ -3,7 +3,7 @@ export function basename(path: string): string {
}
export function embedPreviewKind(
mimeType: string | null,
mimeType: string | undefined,
): "image" | "document" | "video" | "audio" | undefined {
if (mimeType?.startsWith("image")) return "image";
if (mimeType?.startsWith("application")) return "document";
@@ -46,3 +46,24 @@ export function retinaWidth(bytes: Uint8Array): number | undefined {
}
return undefined;
}
export function encodeEmbedUpload(name: string, bytes: Uint8Array): Uint8Array {
const encodedName = new TextEncoder().encode(name);
const body = new Uint8Array(encodedName.length + 1 + bytes.length);
body.set(encodedName);
body.set(bytes, encodedName.length + 1);
return body;
}
export function decodeEmbed(buffer: ArrayBuffer): {
mimeType: string | undefined;
content: Uint8Array<ArrayBuffer>;
} {
const bytes = new Uint8Array(buffer);
const end = bytes.indexOf(0);
const mimeType = new TextDecoder().decode(bytes.subarray(0, end));
return {
mimeType: mimeType === "" ? undefined : mimeType,
content: bytes.subarray(end + 1),
};
}
+14 -4
View File
@@ -69,12 +69,16 @@ async function withTestBackend<T>(
}
});
} else {
const raw = args instanceof Uint8Array;
const headers = new Headers(options?.headers);
headers.set(
"Content-Type",
raw ? "application/octet-stream" : "application/json",
);
return fetch(`http://127.0.0.1:${testHttpApiPort()}/${cmd}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(args ?? {}, (_key, value: unknown) =>
value instanceof Uint8Array ? Array.from(value) : value,
),
headers,
body: raw ? args.slice() : JSON.stringify(args ?? {}),
}).then(async response => {
if (response.status === 404) {
throw new InvokeError(
@@ -82,6 +86,12 @@ async function withTestBackend<T>(
"TestHttpApi.MissingRoute",
);
}
if (
response.ok &&
response.headers.get("Content-Type") === "application/octet-stream"
) {
return response.arrayBuffer();
}
const json = await response.json();
if (!response.ok) {
if (typeof json === "object" && json !== null && "message" in json) {
+4 -6
View File
@@ -1,4 +1,3 @@
import type { EmbedWithMimeType } from "@bindings/cob/EmbedWithMimeType";
import type { JSHandle, Locator, Page } from "@playwright/test";
import type { RadiclePeer } from "@tests/support/peerManager.js";
@@ -119,9 +118,8 @@ test("a pasted image is stored as the image itself", async ({ page, peer }) => {
`http://127.0.0.1:${peer.httpdBaseUrl.port}/get_embed`,
{ data: { rid, oid, name: "image.png" } },
);
const embed = (await response.json()) as EmbedWithMimeType;
expect(embed.mimeType).toBe("image/png");
expect(Buffer.from(embed.content)).toEqual(
Buffer.from(onePixelPng, "base64"),
);
const body = await response.body();
const end = body.indexOf(0);
expect(body.subarray(0, end).toString()).toBe("image/png");
expect(body.subarray(end + 1)).toEqual(Buffer.from(onePixelPng, "base64"));
});
+8 -5
View File
@@ -43,11 +43,14 @@ async function saveEmbed(
name: string,
bytes: Uint8Array,
) {
return (await api(page, peer, "save_embed_by_bytes", {
rid,
name,
bytes: Array.from(bytes),
})) as string;
const response = await page.request.post(
`http://127.0.0.1:${peer.httpdBaseUrl.port}/save_embed_by_bytes`,
{
headers: { rid, "Content-Type": "application/octet-stream" },
data: Buffer.concat([Buffer.from(`${name}\0`), bytes]),
},
);
return (await response.json()) as string;
}
async function createComment(
+32 -2
View File
@@ -1,6 +1,12 @@
import { describe, expect, test } from "vitest";
import { basename, embedPreviewKind, retinaWidth } from "@app/lib/embeds";
import {
basename,
decodeEmbed,
embedPreviewKind,
encodeEmbedUpload,
retinaWidth,
} from "@app/lib/embeds";
describe("basename", () => {
test.each([
@@ -18,7 +24,7 @@ test.each([
["video/mp4", "video"],
["audio/ogg", "audio"],
["text/plain", undefined],
[null, undefined],
[undefined, undefined],
])("a %j embed previews as %j", (mimeType, expected) => {
expect(embedPreviewKind(mimeType)).toBe(expected);
});
@@ -110,3 +116,27 @@ describe("retinaWidth", () => {
expect(retinaWidth(bytes)).toBeUndefined();
});
});
describe("decodeEmbed", () => {
test("splits the MIME type from the content at the first NUL", () => {
const bytes = new Uint8Array([...bytesOf("image/png\0"), 1, 0, 2]);
const { mimeType, content } = decodeEmbed(bytes.buffer);
expect(mimeType).toBe("image/png");
expect(content).toEqual(new Uint8Array([1, 0, 2]));
});
test("reads an empty MIME type as unknown", () => {
const bytes = new Uint8Array([0, 1, 2]);
const { mimeType, content } = decodeEmbed(bytes.buffer);
expect(mimeType).toBeUndefined();
expect(content).toEqual(new Uint8Array([1, 2]));
});
});
test("encodeEmbedUpload puts the UTF-8 name before a NUL and the content", () => {
const name = "Screenshot at 3.07.58\u202fPM.png";
const encodedName = new TextEncoder().encode(name);
expect(encodeEmbedUpload(name, new Uint8Array([1, 0, 2]))).toEqual(
new Uint8Array([...encodedName, 0, 1, 0, 2]),
);
});