Model3D point cloud and wireframe display modes (#8687)

* display modes

* add changeset

* test fixes

* lint

* Update gradio/components/model3d.py

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

* solid

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/wise-animals-go.md

@@ -0,0 +1,6 @@
+---
+"@gradio/model3d": minor
+"gradio": minor
+---
+
+feat:Model3D point cloud and wireframe display modes

demo/model3D/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", "    return mesh_file_name\n", "\n", "\n", "demo = gr.Interface(\n", "    fn=load_mesh,\n", "    inputs=gr.Model3D(),\n", "    outputs=gr.Model3D(\n", "            clear_color=[0.0, 0.0, 0.0, 0.0],  label=\"3D Model\"),\n", "    examples=[\n", "        [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", "        [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", "        [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat\"],\n", "        [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply\"],\n", "    ],\n", "    cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", "    return mesh_file_name\n", "\n", "demo = gr.Interface(\n", "    fn=load_mesh,\n", "    inputs=gr.Model3D(),\n", "    outputs=gr.Model3D(\n", "            clear_color=[0.0, 0.0, 0.0, 0.0],  label=\"3D Model\", display_mode=\"wireframe\"),\n", "    examples=[\n", "        [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", "        [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", "        [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat\"],\n", "        [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply\"],\n", "    ],\n", "    cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/model3D/run.py

@@ -5,12 +5,11 @@
 def load_mesh(mesh_file_name):
     return mesh_file_name
 
-
 demo = gr.Interface(
     fn=load_mesh,
     inputs=gr.Model3D(),
     outputs=gr.Model3D(
-            clear_color=[0.0, 0.0, 0.0, 0.0],  label="3D Model"),
+            clear_color=[0.0, 0.0, 0.0, 0.0],  label="3D Model", display_mode="wireframe"),
     examples=[
         [os.path.join(os.path.dirname(__file__), "files/Bunny.obj")],
         [os.path.join(os.path.dirname(__file__), "files/Duck.glb")],

gradio/components/model3d.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING, Callable
+from typing import TYPE_CHECKING, Callable, Literal
 
 from gradio_client import handle_file
 from gradio_client.documentation import document
@@ -32,6 +32,7 @@ def __init__(
         self,
         value: str | Callable | None = None,
         *,
+        display_mode: Literal["solid", "point_cloud", "wireframe"] | None = None,
         clear_color: tuple[float, float, float, float] | None = None,
         camera_position: tuple[
             int | float | None, int | float | None, int | float | None
@@ -60,6 +61,7 @@ def __init__(
         """
         Parameters:
             value: path to (.obj, .glb, .stl, .gltf, .splat, or .ply) file to show in model3D viewer. If callable, the function will be called whenever the app loads to set the initial value of the component.
+            display_mode: the display mode of the 3D model in the scene. Can be "solid" (which renders the model as a solid object), "point_cloud", or "wireframe". For .splat, or .ply files, this parameter is ignored, as those files can only be rendered as solid objects.
             clear_color: background color of scene, should be a tuple of 4 floats between 0 and 1 representing RGBA values.
             camera_position: initial camera position of scene, provided as a tuple of `(alpha, beta, radius)`. Each value is optional. If provided, `alpha` and `beta` should be in degrees reflecting the angular position along the longitudinal and latitudinal axes, respectively. Radius corresponds to the distance from the center of the object to the camera.
             zoom_speed: the speed of zooming in and out of the scene when the cursor wheel is rotated or when screen is pinched on a mobile device. Should be a positive float, increase this value to make zooming faster, decrease to make it slower. Affects the wheelPrecision property of the camera.
@@ -79,6 +81,7 @@ def __init__(
             render: If False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later.
             key: if assigned, will be used to assume identity across a re-render. Components that have the same key across a re-render will have their value preserved.
         """
+        self.display_mode = display_mode
         self.clear_color = clear_color or [0, 0, 0, 0]
         self.camera_position = camera_position
         self.height = height

js/model3D/Index.svelte

@@ -20,6 +20,7 @@
 	export let visible = true;
 	export let value: null | FileData = null;
 	export let root: string;
+	export let display_mode: "solid" | "point_cloud" | "wireframe" = "solid";
 	export let clear_color: [number, number, number, number];
 	export let loading_status: LoadingStatus;
 	export let label: string;
@@ -66,6 +67,7 @@
 			<Model3D
 				{value}
 				i18n={gradio.i18n}
+				{display_mode}
 				{clear_color}
 				{label}
 				{show_label}
@@ -105,6 +107,7 @@
 			{label}
 			{show_label}
 			{root}
+			{display_mode}
 			{clear_color}
 			{value}
 			{camera_position}

js/model3D/shared/Canvas3D.svelte

@@ -15,6 +15,7 @@
 	}
 
 	export let value: FileData;
+	export let display_mode: "solid" | "point_cloud" | "wireframe";
 	export let clear_color: [number, number, number, number];
 	export let camera_position: [number | null, number | null, number | null];
 	export let zoom_speed: number;
@@ -53,6 +54,7 @@
 	let canvas: HTMLCanvasElement;
 	let scene: BABYLON.Scene;
 	let engine: BABYLON.Engine;
+	let point_cloud_system: BABYLON.PointsCloudSystem | null = null;
 	let mounted = false;
 
 	onMount(() => {
@@ -91,14 +93,27 @@
 				mesh.dispose();
 			});
 
+			if (point_cloud_system) {
+				point_cloud_system.dispose();
+				point_cloud_system = null;
+			}
+
 			// Load the new model
 			if (url) {
 				BABYLON.SceneLoader.ShowLoadingScreen = false;
 				BABYLON.SceneLoader.Append(
 					url,
 					"",
 					scene,
-					() => create_camera(scene, camera_position, zoom_speed, pan_speed),
+					() => {
+						if (display_mode === "point_cloud") {
+							create_point_cloud(scene);
+						} else if (display_mode === "wireframe") {
+							create_wireframe(scene);
+						} else {
+							create_camera(scene, camera_position, zoom_speed, pan_speed);
+						}
+					},
 					undefined,
 					undefined,
 					"." + value.path.split(".").pop()
@@ -145,6 +160,68 @@
 			create_camera(scene, camera_position, zoom_speed, pan_speed);
 		}
 	}
+
+	function create_point_cloud(scene: BABYLON.Scene): void {
+		const meshes = scene.meshes;
+		const pointPositions: BABYLON.Vector3[] = [];
+
+		meshes.forEach((mesh) => {
+			if (mesh instanceof BABYLON.Mesh) {
+				const positions = mesh.getVerticesData(
+					BABYLON.VertexBuffer.PositionKind
+				);
+				if (positions) {
+					for (let i = 0; i < positions.length; i += 3) {
+						pointPositions.push(
+							new BABYLON.Vector3(
+								positions[i],
+								positions[i + 1],
+								positions[i + 2]
+							)
+						);
+					}
+				}
+				mesh.setEnabled(false);
+			}
+		});
+
+		point_cloud_system = new BABYLON.PointsCloudSystem(
+			"point_cloud_system",
+			1,
+			scene
+		);
+
+		point_cloud_system.addPoints(
+			pointPositions.length,
+			(particle: BABYLON.CloudPoint, i: number) => {
+				particle.position = pointPositions[i];
+				particle.color = new BABYLON.Color4(
+					Math.random(),
+					Math.random(),
+					Math.random(),
+					1.0
+				);
+			}
+		);
+
+		point_cloud_system.buildMeshAsync().then((mesh) => {
+			mesh.alwaysSelectAsActiveMesh = true;
+			create_camera(scene, camera_position, zoom_speed, pan_speed);
+		});
+	}
+
+	function create_wireframe(scene: BABYLON.Scene): void {
+		scene.meshes.forEach((mesh) => {
+			if (mesh instanceof BABYLON.Mesh) {
+				mesh.material = new BABYLON.StandardMaterial(
+					"wireframeMaterial",
+					scene
+				);
+				mesh.material.wireframe = true;
+			}
+			create_camera(scene, camera_position, zoom_speed, pan_speed);
+		});
+	}
 </script>
 
 <canvas bind:this={canvas}></canvas>

js/model3D/shared/Model3D.svelte

@@ -8,6 +8,7 @@
 	import type Canvas3D from "./Canvas3D.svelte";
 
 	export let value: FileData | null;
+	export let display_mode: "solid" | "point_cloud" | "wireframe" = "solid";
 	export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
 	export let label = "";
 	export let show_label: boolean;
@@ -101,6 +102,7 @@
 				bind:this={canvas3d}
 				bind:resolved_url
 				{value}
+				{display_mode}
 				{clear_color}
 				{camera_position}
 				{zoom_speed}

js/model3D/shared/Model3DUpload.svelte

@@ -9,6 +9,7 @@
 	import type Canvas3D from "./Canvas3D.svelte";
 
 	export let value: null | FileData;
+	export let display_mode: "solid" | "point_cloud" | "wireframe" = "solid";
 	export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
 	export let label = "";
 	export let show_label: boolean;
@@ -121,6 +122,7 @@
 				this={Canvas3DComponent}
 				bind:this={canvas3d}
 				{value}
+				{display_mode}
 				{clear_color}
 				{camera_position}
 				{zoom_speed}

test/components/test_model3d.py

@@ -11,6 +11,7 @@ def test_component_functions(self):
         model_component = gr.components.Model3D(None, label="Model")
         assert model_component.get_config() == {
             "value": None,
+            "display_mode": None,
             "clear_color": [0, 0, 0, 0],
             "label": "Model",
             "show_label": True,