Apple Vision · ScenePrint revision 2

網頁能不能呼叫你本機的 Vision

這一頁是公開的 HTTPS 網站,你的 Mac 上跑一支本機小程式。 網頁把照片送給它,它呼叫 Apple Vision 算出 768 維向量再回傳。 照片不會離開你的機器。這正是那個架構問題的真實測試場景。

00先在你的 Mac 上跑起 bridge

單檔 Swift,沒有專案檔、沒有相依套件、沒有簽章、不需要開發者帳號。 swiftc 是 Xcode Command Line Tools 內建的。

# 存成 VisionBridge.swift 之後
swiftc -O VisionBridge.swift -o visionbridge
./visionbridge
# vision bridge on http://127.0.0.1:8787
看原始碼(96 行)
import Foundation
import Vision
import Network

let port: NWEndpoint.Port = 8787

func featurePrint(jpeg: Data) throws -> [Float] {
    let r = VNGenerateImageFeaturePrintRequest()
    r.revision = VNGenerateImageFeaturePrintRequestRevision2
    try VNImageRequestHandler(data: jpeg).perform([r])
    guard let d = r.results?.first?.data else { return [] }
    return d.withUnsafeBytes { Array($0.bindMemory(to: Float.self)) }
}
func featurePrint(path: String) throws -> [Float] {
    let r = VNGenerateImageFeaturePrintRequest()
    r.revision = VNGenerateImageFeaturePrintRequestRevision2
    try VNImageRequestHandler(url: URL(fileURLWithPath: path)).perform([r])
    guard let d = r.results?.first?.data else { return [] }
    return d.withUnsafeBytes { Array($0.bindMemory(to: Float.self)) }
}

func respond(_ conn: NWConnection, status: String, body: Data, type: String = "application/json") {
    var head = "HTTP/1.1 \(status)\r\n"
    head += "Content-Type: \(type)\r\n"
    head += "Content-Length: \(body.count)\r\n"
    head += "Access-Control-Allow-Origin: *\r\n"
    head += "Access-Control-Allow-Headers: Content-Type\r\n"
    head += "Access-Control-Allow-Private-Network: true\r\n"
    head += "Connection: close\r\n\r\n"
    conn.send(content: Data(head.utf8) + body, completion: .contentProcessed { _ in conn.cancel() })
}

func handle(_ conn: NWConnection, _ raw: Data) {
    guard let sep = raw.range(of: Data("\r\n\r\n".utf8)) else { return }
    let head = String(decoding: raw[..<sep.lowerBound], as: UTF8.self)
    let body = raw[sep.upperBound...]
    let line = head.split(separator: "\r\n").first.map(String.init) ?? ""
    let parts = line.split(separator: " ")
    let method = parts.first.map(String.init) ?? ""
    let target = parts.count > 1 ? String(parts[1]) : "/"
    FileHandle.standardError.write(Data("\(method) \(target)\n".utf8))
    if method == "OPTIONS" { respond(conn, status: "204 No Content", body: Data()); return }
    do {
        if method == "GET", target.hasPrefix("/embed?path=") {
            let p = String(target.dropFirst("/embed?path=".count)).removingPercentEncoding ?? ""
            let v = try featurePrint(path: p)
            respond(conn, status: "200 OK",
                    body: try JSONSerialization.data(withJSONObject: ["dims": v.count, "revision": 2, "vector": v]))
        } else if method == "POST", target == "/embed" {
            let v = try featurePrint(jpeg: Data(body))
            respond(conn, status: "200 OK",
                    body: try JSONSerialization.data(withJSONObject: ["dims": v.count, "revision": 2, "vector": v]))
        } else if target == "/health" {
            respond(conn, status: "200 OK", body: Data(#"{"ok":true,"revision":2,"dims":768}"#.utf8))
        } else {
            respond(conn, status: "404 Not Found", body: Data(#"{"error":"no such route"}"#.utf8))
        }
    } catch {
        respond(conn, status: "500 Internal Server Error",
                body: (try? JSONSerialization.data(withJSONObject: ["error": "\(error)"])) ?? Data())
    }
}

let params = NWParameters.tcp
params.requiredLocalEndpoint = NWEndpoint.hostPort(host: "127.0.0.1", port: port)
let listener = try NWListener(using: params)
listener.newConnectionHandler = { conn in
    conn.start(queue: .global())
    var buf = Data()
    func read() {
        conn.receive(minimumIncompleteLength: 1, maximumLength: 1 << 20) { d, _, done, _ in
            if let d { buf.append(d) }
            let hasHead = buf.range(of: Data("\r\n\r\n".utf8)) != nil
            var complete = done
            if hasHead, let sep = buf.range(of: Data("\r\n\r\n".utf8)) {
                let h = String(decoding: buf[..<sep.lowerBound], as: UTF8.self)
                if let m = h.range(of: "Content-Length: ", options: .caseInsensitive) {
                    let n = Int(h[m.upperBound...].prefix(while: \.isNumber)) ?? 0
                    complete = buf.count - sep.upperBound >= n
                } else { complete = true }
            }
            if complete && hasHead { handle(conn, buf) } else if !done { read() }
        }
    }
    read()
}
listener.start(queue: .main)
FileHandle.standardError.write(Data("vision bridge on http://127.0.0.1:\(port)\n".utf8))
RunLoop.main.run()

01丟照片進來

照片以原始檔案位元組送出,這條路徑跟本機直接讀檔算出來的向量逐位相同(41/41 實測)。 丟兩張以上會算出彼此的 cosine 相似度。

把照片拖進來

02原始位元組 vs 瀏覽器先解碼

同一張照片送兩次:一次直接送檔案位元組,一次先畫進 canvas 再重新編碼送出。 兩者是同一張照片,看向量差多少。需要先在上面丟至少一張。

03剛才發生了什麼