| | <!doctype html> |
| | <html lang="en"> |
| |
|
| | <head> |
| | <meta charset="UTF-8" /> |
| | <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| | <title>SAM3 WebGPU | Transformers.js</title> |
| |
|
| | <script src="https://cdn.tailwindcss.com"></script> |
| |
|
| | <style> |
| | @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap'); |
| | |
| | body { |
| | font-family: 'Inter', sans-serif; |
| | } |
| | |
| | |
| | canvas { |
| | position: absolute; |
| | top: 0; |
| | left: 0; |
| | opacity: 0.6; |
| | pointer-events: none; |
| | } |
| | |
| | |
| | .icon { |
| | position: absolute; |
| | transform: translate(-50%, -50%); |
| | font-size: 24px; |
| | user-select: none; |
| | pointer-events: none; |
| | text-shadow: 0 0 4px white; |
| | } |
| | |
| | .aspect-w-3 { |
| | position: relative; |
| | width: 100%; |
| | } |
| | |
| | .aspect-w-3::before { |
| | content: ''; |
| | display: block; |
| | padding-bottom: calc(var(--aspect-h) / var(--aspect-w) * 100%); |
| | } |
| | |
| | .aspect-w-3> :first-child { |
| | position: absolute; |
| | top: 0; |
| | left: 0; |
| | width: 100%; |
| | height: 100%; |
| | } |
| | |
| | .aspect-w-3 { |
| | --aspect-w: 3; |
| | } |
| | |
| | .aspect-h-2 { |
| | --aspect-h: 2; |
| | } |
| | |
| | .aspect-w-4 { |
| | --aspect-w: 4; |
| | } |
| | |
| | .aspect-h-3 { |
| | --aspect-h: 3; |
| | } |
| | </style> |
| | </head> |
| |
|
| | <body class="bg-gray-100 text-gray-800 min-h-screen flex flex-col items-center justify-center p-4 sm:p-8"> |
| |
|
| | <div class="w-full max-w-3xl bg-white rounded-xl shadow-2xl overflow-hidden"> |
| |
|
| | <div class="p-6 sm:p-10"> |
| | <h1 class="text-3xl sm:text-4xl font-bold text-center text-gray-900">SAM3 WebGPU</h1> |
| | <h3 class="text-lg sm:text-xl text-gray-500 text-center mb-6"> |
| | In-browser image segmentation w/ |
| | <a href="https://hf.co/docs/transformers.js" target="_blank" class="text-blue-600 hover:underline">🤗 |
| | Transformers.js</a> |
| | </h3> |
| |
|
| | <div id="container" |
| | class="relative w-full max-w-2xl mx-auto border border-gray-200 rounded-lg overflow-hidden cursor-pointer bg-gray-50 shadow-sm transition-all aspect-w-3 aspect-h-2"> |
| |
|
| | <label id="upload-area" for="upload" |
| | class="absolute inset-0 z-10 flex flex-col justify-center items-center p-10 transition-all cursor-pointer border-2 border-dashed border-gray-300 rounded-lg hover:bg-gray-50/50 hover:border-blue-500"> |
| | <div class="flex flex-col items-center justify-center p-6 transition-colors w-full max-w-sm"> |
| | <svg class="w-12 h-12 text-gray-400" fill="currentColor" viewBox="0 0 25 25" |
| | xmlns="http://www.w3.org/2000/svg"> |
| | <path |
| | d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z"> |
| | </path> |
| | </svg> |
| | <span class="text-lg font-medium text-gray-500 mt-2">Click to upload image</span> |
| | <span class="text-sm text-gray-400">or drag and drop</span> |
| | </div> |
| |
|
| | <p class="text-gray-500 text-sm my-4">...or try an example:</p> |
| |
|
| | <div id="example-gallery" class="flex gap-4"> |
| | <img src="https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg" |
| | class="example-image w-20 h-20 sm:w-24 sm:h-24 object-cover rounded-lg shadow-md cursor-pointer hover:opacity-80 transition-opacity" |
| | alt="Example of a truck"> |
| | <img src="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/corgi.jpg" |
| | class="example-image w-20 h-20 sm:w-24 sm:h-24 object-cover rounded-lg shadow-md cursor-pointer hover:opacity-80 transition-opacity" |
| | alt="Example of a corgi"> |
| | <img src="https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/groceries.jpg" |
| | class="example-image w-20 h-20 sm:w-24 sm:h-24 object-cover rounded-lg shadow-md cursor-pointer hover:opacity-80 transition-opacity" |
| | alt="Example of groceries"> |
| | </div> |
| | </label> |
| |
|
| | <img id="image-display" class="absolute inset-0 w-full h-full object-contain block hidden z-0" /> |
| |
|
| | <canvas id="mask-output"></canvas> |
| | </div> |
| |
|
| | <label id="status" class="text-base text-center text-gray-600 min-h-[1.5rem] mt-6 mb-4 block w-full">Loading |
| | model...</label> |
| |
|
| | <div id="controls" class="flex flex-col sm:flex-row justify-center gap-3"> |
| | <button id="reset-image" disabled |
| | class="w-full sm:w-auto inline-flex items-center justify-center px-4 py-2 bg-gray-200 text-gray-800 font-medium rounded-lg shadow-sm hover:bg-gray-300 transition-colors disabled:bg-gray-100 disabled:text-gray-400 disabled:cursor-not-allowed"> |
| | <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
| | stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" |
| | class="w-4 h-4 mr-2"> |
| | <path d="M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8" /> |
| | <path d="M3 3v5h5" /> |
| | </svg> |
| | Reset image |
| | </button> |
| | <button id="clear-points" disabled |
| | class="w-full sm:w-auto inline-flex items-center justify-center px-4 py-2 bg-gray-200 text-gray-800 font-medium rounded-lg shadow-sm hover:bg-gray-300 transition-colors disabled:bg-gray-100 disabled:text-gray-400 disabled:cursor-not-allowed"> |
| | <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
| | stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" |
| | class="w-4 h-4 mr-2"> |
| | <line x1="18" y1="6" x2="6" y2="18"></line> |
| | <line x1="6" y1="6" x2="18" y2="18"></line> |
| | </svg> |
| | Clear points |
| | </button> |
| | <button id="cut-mask" disabled |
| | class="w-full sm:w-auto inline-flex items-center justify-center px-4 py-2 bg-blue-600 text-white font-medium rounded-lg shadow-sm hover:bg-blue-700 transition-colors disabled:bg-gray-300 disabled:text-gray-500 disabled:cursor-not-allowed"> |
| | <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" |
| | stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" |
| | class="w-4 h-4 mr-2"> |
| | <path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4" /> |
| | <polyline points="7 10 12 15 17 10" /> |
| | <line x1="12" y1="15" x2="12" y2="3" /> |
| | </svg> |
| | Cut & Download |
| | </button> |
| | </div> |
| |
|
| | <p id="information" class="text-sm text-gray-500 mt-4 text-center"> |
| | Left click = positive (⭐), Right click = negative (❌). |
| | </p> |
| | </div> |
| | </div> |
| |
|
| | <input id="upload" type="file" accept="image/*" disabled class="hidden" /> |
| |
|
| |
|
| | <script type="module"> |
| | import { |
| | Sam3TrackerModel, |
| | AutoProcessor, |
| | RawImage, |
| | Tensor, |
| | } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@next"; |
| | |
| | const statusLabel = document.getElementById("status"); |
| | const fileUpload = document.getElementById("upload"); |
| | const imageContainer = document.getElementById("container"); |
| | const uploadArea = document.getElementById("upload-area"); |
| | const exampleImages = document.querySelectorAll(".example-image"); |
| | const resetButton = document.getElementById("reset-image"); |
| | const clearButton = document.getElementById("clear-points"); |
| | const cutButton = document.getElementById("cut-mask"); |
| | const imageDisplay = document.getElementById("image-display"); |
| | const maskCanvas = document.getElementById("mask-output"); |
| | const maskContext = maskCanvas.getContext("2d"); |
| | |
| | let isEncoding = false; |
| | let isDecoding = false; |
| | let decodePending = false; |
| | let lastPoints = null; |
| | let isMultiMaskMode = false; |
| | let imageInput = null; |
| | let imageProcessed = null; |
| | let imageEmbeddings = null; |
| | |
| | let model, processor; |
| | |
| | |
| | |
| | |
| | async function encode(url) { |
| | if (isEncoding || !processor || !model) return; |
| | isEncoding = true; |
| | statusLabel.textContent = "Extracting image embedding..."; |
| | |
| | try { |
| | imageInput = await RawImage.fromURL(url); |
| | |
| | imageDisplay.onload = updateCanvasGeometry; |
| | imageDisplay.src = url; |
| | imageDisplay.classList.remove('hidden'); |
| | uploadArea.classList.add("hidden"); |
| | cutButton.disabled = true; |
| | |
| | imageProcessed = await processor(imageInput); |
| | imageEmbeddings = await model.get_image_embeddings(imageProcessed); |
| | console.log({ imageEmbeddings }) |
| | |
| | statusLabel.textContent = "Embedding extracted! Click on the image."; |
| | resetButton.disabled = false; |
| | clearButton.disabled = false; |
| | } catch (error) { |
| | console.error("Error during encoding:", error); |
| | statusLabel.textContent = "Error loading image. Please try again."; |
| | resetUI(); |
| | } finally { |
| | isEncoding = false; |
| | } |
| | } |
| | |
| | |
| | |
| | |
| | async function decode() { |
| | if (isDecoding || !imageEmbeddings || !lastPoints || lastPoints.length === 0) { |
| | if (isDecoding) { |
| | decodePending = true; |
| | } |
| | return; |
| | } |
| | isDecoding = true; |
| | |
| | try { |
| | const reshaped = imageProcessed.reshaped_input_sizes[0]; |
| | const points = lastPoints |
| | .map((x) => [x.position[0] * reshaped[1], x.position[1] * reshaped[0]]) |
| | .flat(Infinity); |
| | const labels = lastPoints.map((x) => BigInt(x.label)).flat(Infinity); |
| | |
| | const num_points = lastPoints.length; |
| | const input_points = new Tensor("float32", points, [1, 1, num_points, 2]); |
| | const input_labels = new Tensor("int64", labels, [1, 1, num_points]); |
| | |
| | const { pred_masks, iou_scores } = await model({ |
| | ...imageEmbeddings, |
| | input_points, |
| | input_labels, |
| | }); |
| | |
| | const masks = await processor.post_process_masks( |
| | pred_masks, |
| | imageProcessed.original_sizes, |
| | imageProcessed.reshaped_input_sizes, |
| | ); |
| | |
| | updateMaskOverlay(RawImage.fromTensor(masks[0][0]), iou_scores.data); |
| | |
| | } catch (error) { |
| | console.error("Error during decoding:", error); |
| | statusLabel.textContent = "Error generating mask."; |
| | } finally { |
| | isDecoding = false; |
| | } |
| | |
| | if (decodePending) { |
| | decodePending = false; |
| | setTimeout(decode, 0); |
| | } |
| | } |
| | |
| | |
| | |
| | |
| | function updateCanvasGeometry() { |
| | if (!imageDisplay.src || imageDisplay.classList.contains('hidden')) return; |
| | |
| | const { naturalWidth, naturalHeight } = imageDisplay; |
| | const { width: containerWidth, height: containerHeight } = imageContainer.getBoundingClientRect(); |
| | |
| | const imageAspectRatio = naturalWidth / naturalHeight; |
| | const containerAspectRatio = containerWidth / containerHeight; |
| | |
| | let newWidth, newHeight, newTop, newLeft; |
| | |
| | if (imageAspectRatio > containerAspectRatio) { |
| | newWidth = containerWidth; |
| | newHeight = newWidth / imageAspectRatio; |
| | newTop = (containerHeight - newHeight) / 2; |
| | newLeft = 0; |
| | } else { |
| | newHeight = containerHeight; |
| | newWidth = newHeight * imageAspectRatio; |
| | newLeft = (containerWidth - newWidth) / 2; |
| | newTop = 0; |
| | } |
| | |
| | maskCanvas.style.width = `${newWidth}px`; |
| | maskCanvas.style.height = `${newHeight}px`; |
| | maskCanvas.style.top = `${newTop}px`; |
| | maskCanvas.style.left = `${newLeft}px`; |
| | } |
| | |
| | |
| | |
| | |
| | function updateMaskOverlay(mask, scores) { |
| | if (maskCanvas.width !== mask.width || maskCanvas.height !== mask.height) { |
| | maskCanvas.width = mask.width; |
| | maskCanvas.height = mask.height; |
| | } |
| | |
| | const imageData = maskContext.createImageData( |
| | maskCanvas.width, |
| | maskCanvas.height, |
| | ); |
| | |
| | const numMasks = scores.length; |
| | let bestIndex = 0; |
| | for (let i = 1; i < numMasks; ++i) { |
| | if (scores[i] > scores[bestIndex]) { |
| | bestIndex = i; |
| | } |
| | } |
| | statusLabel.textContent = `Segment score: ${scores[bestIndex].toFixed(2)}`; |
| | |
| | const pixelData = imageData.data; |
| | for (let i = 0; i < pixelData.length; ++i) { |
| | if (mask.data[numMasks * i + bestIndex] === 1) { |
| | const offset = 4 * i; |
| | pixelData[offset] = 0; |
| | pixelData[offset + 1] = 114; |
| | pixelData[offset + 2] = 189; |
| | pixelData[offset + 3] = 255; |
| | } |
| | } |
| | |
| | maskContext.putImageData(imageData, 0, 0); |
| | } |
| | |
| | function clearPointsAndMask() { |
| | isMultiMaskMode = false; |
| | lastPoints = null; |
| | |
| | document.querySelectorAll(".icon").forEach((e) => e.remove()); |
| | |
| | cutButton.disabled = true; |
| | |
| | maskContext.clearRect(0, 0, maskCanvas.width, maskCanvas.height); |
| | statusLabel.textContent = "Points cleared. Click to add new points."; |
| | } |
| | |
| | function resetUI() { |
| | imageInput = null; |
| | imageProcessed = null; |
| | imageEmbeddings = null; |
| | isEncoding = false; |
| | isDecoding = false; |
| | decodePending = false; |
| | |
| | clearPointsAndMask(); |
| | |
| | cutButton.disabled = true; |
| | resetButton.disabled = true; |
| | clearButton.disabled = true; |
| | imageDisplay.src = ''; |
| | imageDisplay.classList.add('hidden'); |
| | uploadArea.classList.remove("hidden"); |
| | |
| | |
| | maskCanvas.style.width = '0px'; |
| | maskCanvas.style.height = '0px'; |
| | |
| | exampleImages.forEach(img => img.style.pointerEvents = "auto"); |
| | |
| | statusLabel.textContent = "Ready"; |
| | } |
| | function clamp(x, min = 0, max = 1) { |
| | return Math.max(Math.min(x, max), min); |
| | } |
| | function getPoint(e) { |
| | const imgBB = imageDisplay.getBoundingClientRect(); |
| | const canvasBB = maskCanvas.getBoundingClientRect(); |
| | |
| | |
| | const mouseX = clamp((e.clientX - canvasBB.left) / canvasBB.width); |
| | const mouseY = clamp((e.clientY - canvasBB.top) / canvasBB.height); |
| | |
| | return { |
| | position: [mouseX, mouseY], |
| | label: e.button === 2 ? 0 : 1, |
| | }; |
| | } |
| | |
| | fileUpload.addEventListener("change", function (e) { |
| | const file = e.target.files[0]; |
| | if (!file) return; |
| | |
| | const reader = new FileReader(); |
| | reader.onload = (e2) => encode(e2.target.result); |
| | reader.readAsDataURL(file); |
| | }); |
| | |
| | exampleImages.forEach((img) => { |
| | img.addEventListener("click", (e) => { |
| | e.preventDefault(); |
| | e.stopPropagation(); |
| | exampleImages.forEach(i => i.style.pointerEvents = "none"); |
| | encode(img.src); |
| | }); |
| | }); |
| | |
| | window.addEventListener("resize", updateCanvasGeometry); |
| | |
| | resetButton.addEventListener("click", resetUI); |
| | |
| | clearButton.addEventListener("click", clearPointsAndMask); |
| | |
| | imageContainer.addEventListener("mousedown", (e) => { |
| | if (!imageEmbeddings || uploadArea.classList.contains('hidden') === false) { |
| | return; |
| | } |
| | |
| | if (e.button !== 0 && e.button !== 2) return; |
| | |
| | if (!isMultiMaskMode) { |
| | lastPoints = []; |
| | isMultiMaskMode = true; |
| | cutButton.disabled = false; |
| | } |
| | |
| | const point = getPoint(e); |
| | lastPoints.push(point); |
| | |
| | const icon = document.createElement('span'); |
| | icon.className = 'icon'; |
| | icon.textContent = point.label === 1 ? '⭐' : '❌'; |
| | |
| | |
| | const canvasRect = maskCanvas.getBoundingClientRect(); |
| | const containerRect = imageContainer.getBoundingClientRect(); |
| | const left = canvasRect.left - containerRect.left + point.position[0] * canvasRect.width; |
| | const top = canvasRect.top - containerRect.top + point.position[1] * canvasRect.height; |
| | |
| | icon.style.left = `${left}px`; |
| | icon.style.top = `${top}px`; |
| | imageContainer.appendChild(icon); |
| | |
| | decode(); |
| | }); |
| | |
| | imageContainer.addEventListener("contextmenu", (e) => e.preventDefault()); |
| | |
| | imageContainer.addEventListener("mousemove", (e) => { |
| | if (!imageEmbeddings || isMultiMaskMode || uploadArea.classList.contains('hidden') === false) { |
| | return; |
| | } |
| | lastPoints = [getPoint(e)]; |
| | decode(); |
| | }); |
| | |
| | cutButton.addEventListener("click", async () => { |
| | if (!imageInput || !maskCanvas) return; |
| | |
| | const [w, h] = [maskCanvas.width, maskCanvas.height]; |
| | |
| | const maskImageData = maskContext.getImageData(0, 0, w, h); |
| | const maskPixelData = maskImageData.data; |
| | |
| | const cutCanvas = new OffscreenCanvas(w, h); |
| | const cutContext = cutCanvas.getContext("2d"); |
| | |
| | const cutImageData = cutContext.createImageData(w, h); |
| | const cutPixelData = cutImageData.data; |
| | |
| | const imagePixelData = imageInput.data; |
| | |
| | for (let i = 0; i < w * h; ++i) { |
| | const maskOffset = 4 * i; |
| | const imageOffset = 3 * i; |
| | |
| | if (maskPixelData[maskOffset + 3] > 0) { |
| | cutPixelData[maskOffset] = imagePixelData[imageOffset]; |
| | cutPixelData[maskOffset + 1] = imagePixelData[imageOffset + 1]; |
| | cutPixelData[maskOffset + 2] = imagePixelData[imageOffset + 2]; |
| | cutPixelData[maskOffset + 3] = 255; |
| | } |
| | } |
| | cutContext.putImageData(cutImageData, 0, 0); |
| | |
| | const link = document.createElement("a"); |
| | link.download = "mask-cutout.png"; |
| | link.href = URL.createObjectURL(await cutCanvas.convertToBlob()); |
| | link.click(); |
| | link.remove(); |
| | }); |
| | |
| | async function loadModel() { |
| | try { |
| | const model_id = "onnx-community/sam3-tracker-ONNX"; |
| | |
| | model = await Sam3TrackerModel.from_pretrained(model_id, { |
| | dtype: { |
| | vision_encoder: "q4", |
| | prompt_encoder_mask_decoder: "fp32", |
| | }, |
| | device: "webgpu", |
| | }); |
| | |
| | processor = await AutoProcessor.from_pretrained(model_id); |
| | } catch (error) { |
| | console.error("Error loading model:", error); |
| | statusLabel.textContent = "Error loading model. Please refresh the page."; |
| | } |
| | } |
| | |
| | await loadModel(); |
| | fileUpload.disabled = false; |
| | statusLabel.textContent = "Ready"; |
| | |
| | </script> |
| | </body> |
| |
|
| | </html> |
