Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Video widget #51

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

[WIP] Video widget #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a7d2fef
animation for edit widget sliders
w-m Oct 31, 2024
72a792a
preset for deconstructing Self-Organizing Gaussians
w-m Oct 31, 2024
594ffa7
video recording widget (WIP)
w-m Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 235 additions & 1 deletion presets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,235 @@
{"Default": {"edit_text": "gs._xyz = gs._xyz\ngs._rotation = gs._rotation\ngs._scaling = gs._scaling\ngs._opacity = gs._opacity\ngs._features_dc = gs._features_dc\ngs._features_rest = gs._features_rest\n", "slider": [{"key": "x", "value": 1, "min_value": 0, "max_value": 10, "_id": "a8f2a853-17ba-4f7f-9dd8-843d2f0756a1"}]}, "Point Cloud": {"edit_text": "gs._scaling = gs._scaling * 0 - slider.scaling\ngs._opacity = gs._opacity * 0 + 10\n", "slider": [{"key": "scaling", "value": 8.0, "min_value": 1, "max_value": 10, "_id": "6dd9eb75-6bce-445b-9165-dc8dd498d253"}]}, "Only SH": {"edit_text": "gs._features_dc = gs._features_dc * slider.dc_scale\ngs._features_rest = gs._features_rest * slider.sh_scale\n", "slider": [{"key": "dc_scale", "value": 0.0, "min_value": -3, "max_value": 3, "_id": "3c0f6f8c-516b-4303-957e-65a3bebbd8bd"}, {"key": "sh_scale", "value": 1.0, "min_value": -3, "max_value": 3, "_id": "4ae54ccd-b28c-49d9-a514-fd7d6fdfb831"}]}, "Filter Opacity": {"edit_text": "mask = torch.linalg.norm(gs._opacity, dim=-1) < slider.opacity\n\ngs._xyz = gs._xyz[mask]\ngs._rotation = gs._rotation[mask]\ngs._scaling = gs._scaling[mask]\ngs._opacity = gs._opacity[mask]\ngs._features_dc = gs._features_dc[mask]\ngs._features_rest = gs._features_rest[mask]\n", "slider": [{"key": "opacity", "value": 10.0, "min_value": 0, "max_value": 20, "_id": "12364012634"}]}, "Highlight Large gs": {"edit_text": "mask = torch.linalg.norm(gs._scaling, dim=-1) < slider.scaling\n\ncolor = [slider.r, slider.g ,slider.b]\ngs._features_dc[mask] = gs._features_dc[mask] * 0 + torch.tensor(color, device=\"cuda\")\n\n", "slider": [{"key": "scaling", "value": 5.0, "min_value": -10, "max_value": 10, "_id": "bcb1fb24-fae7-4bde-b9f1-3368a3ccf6bd"}, {"key": "r", "value": 1.0, "min_value": -1, "max_value": 1, "_id": "610356120935721"}, {"key": "g", "value": 0.0, "min_value": -1, "max_value": 1, "_id": "19161646843249846"}, {"key": "b", "value": 0.0, "min_value": -1, "max_value": 1, "_id": "12+92+17+174+9174+5"}]}, "Sin(xyz)": {"edit_text": "gs._xyz = torch.sin(gs._xyz/3+slider.x) + gs._xyz\n", "slider": [{"key": "x", "value": 0.0, "min_value": -10, "max_value": 10, "_id": "28971294217951247"}]}, "Random Colors": {"edit_text": "torch.manual_seed(0)\ngs._features_dc = torch.rand_like(gs._features_dc) * slider.intensity - slider.intensity / 2\ngs._features_rest = gs._features_rest * 0\n\n", "slider": [{"key": "intensity", "value": 5.0, "min_value": 0, "max_value": 10, "_id": "9f3fd0c1-67a5-4b38-9ec4-36d27aceddda"}]}, "Recoloring": {"edit_text": "selected_color = torch.tensor([slider.r, slider.g, slider.b], device=\"cuda\")\n\ngs._xyz = gs._xyz\ngs._rotation = gs._rotation\ngs._scaling = gs._scaling\ngs._opacity = gs._opacity\ngs._features_dc = gs._features_dc + selected_color\ngs._features_rest = gs._features_rest\n", "slider": [{"key": "r", "value": 0.0, "min_value": -10, "max_value": 10, "_id": "9419751979747546"}, {"key": "g", "value": 0.0, "min_value": -10, "max_value": 10, "_id": "52795+7+5+4959597"}, {"key": "b", "value": 0.0, "min_value": -10, "max_value": 10, "_id": "47954795627648945"}]}, "Filter Small gs": {"edit_text": "mask = torch.linalg.norm(gs._scaling, dim=-1) < slider.scaling\n\ngs._xyz = gs._xyz[mask]\ngs._rotation = gs._rotation[mask]\ngs._scaling = gs._scaling[mask]\ngs._opacity = gs._opacity[mask]\ngs._features_dc = gs._features_dc[mask]\ngs._features_rest = gs._features_rest[mask]\n", "slider": [{"key": "scaling", "value": 10.0, "min_value": 0, "max_value": 20, "_id": "8769fe63-c073-4b0f-b83d-dc519024a0bb"}]}, "Wobbel": {"edit_text": "gs._xyz[:,2] = torch.sin(gs._xyz[:,2]+slider.x)/5 + gs._xyz[:,2]\ngs._xyz[:,0] = torch.sin(gs._xyz[:,0]+slider.x)/5 + gs._xyz[:,0]", "slider": [{"key": "x", "value": -10.0, "min_value": -10.0, "max_value": 10.0, "_id": "28971294217951247"}]}, "Slice Scene": {"edit_text": "mask_x = gs._xyz[:, 0] < slider.x\nmask_y =gs._xyz[:, 1] < slider.y\nmask_z =gs._xyz[:, 2] < slider.z\n\nmask = mask_x & mask_y & mask_z\n\ngs._xyz = gs._xyz[mask]\ngs._rotation = gs._rotation[mask]\ngs._scaling = gs._scaling[mask]\ngs._opacity = gs._opacity[mask]\ngs._features_dc = gs._features_dc[mask]\ngs._features_rest = gs._features_rest[mask]\n", "slider": [{"key": "x", "value": -0.0010000000474974513, "min_value": -5.0, "max_value": 5.0, "_id": "12364012634"}, {"key": "y", "value": 0.0, "min_value": -5.0, "max_value": 5.0, "_id": "197e4266-82ba-417d-b276-536e465f4fc0"}, {"key": "z", "value": -0.0, "min_value": -5.0, "max_value": 5.0, "_id": "636299d8-5d63-4e7c-8ce4-d192aa019d95"}]}}
{
"Default": {
"edit_text": "gs._xyz = gs._xyz\ngs._rotation = gs._rotation\ngs._scaling = gs._scaling\ngs._opacity = gs._opacity\ngs._features_dc = gs._features_dc\ngs._features_rest = gs._features_rest\n",
"slider": [
{
"key": "x",
"value": 1,
"min_value": 0,
"max_value": 10,
"_id": "a8f2a853-17ba-4f7f-9dd8-843d2f0756a1"
}
]
},
"Point Cloud": {
"edit_text": "gs._scaling = gs._scaling * 0 - slider.scaling\ngs._opacity = gs._opacity * 0 + 10\n",
"slider": [
{
"key": "scaling",
"value": 8.0,
"min_value": 1,
"max_value": 10,
"_id": "6dd9eb75-6bce-445b-9165-dc8dd498d253"
}
]
},
"Only SH": {
"edit_text": "gs._features_dc = gs._features_dc * slider.dc_scale\ngs._features_rest = gs._features_rest * slider.sh_scale\n",
"slider": [
{
"key": "dc_scale",
"value": 0.0,
"min_value": -3,
"max_value": 3,
"_id": "3c0f6f8c-516b-4303-957e-65a3bebbd8bd"
},
{
"key": "sh_scale",
"value": 1.0,
"min_value": -3,
"max_value": 3,
"_id": "4ae54ccd-b28c-49d9-a514-fd7d6fdfb831"
}
]
},
"Filter Opacity": {
"edit_text": "mask = torch.linalg.norm(gs._opacity, dim=-1) < slider.opacity\n\ngs._xyz = gs._xyz[mask]\ngs._rotation = gs._rotation[mask]\ngs._scaling = gs._scaling[mask]\ngs._opacity = gs._opacity[mask]\ngs._features_dc = gs._features_dc[mask]\ngs._features_rest = gs._features_rest[mask]\n",
"slider": [
{
"key": "opacity",
"value": 10.0,
"min_value": 0,
"max_value": 20,
"_id": "12364012634"
}
]
},
"Highlight Large gs": {
"edit_text": "mask = torch.linalg.norm(gs._scaling, dim=-1) < slider.scaling\n\ncolor = [slider.r, slider.g ,slider.b]\ngs._features_dc[mask] = gs._features_dc[mask] * 0 + torch.tensor(color, device=\"cuda\")\n\n",
"slider": [
{
"key": "scaling",
"value": 5.0,
"min_value": -10,
"max_value": 10,
"_id": "bcb1fb24-fae7-4bde-b9f1-3368a3ccf6bd"
},
{
"key": "r",
"value": 1.0,
"min_value": -1,
"max_value": 1,
"_id": "610356120935721"
},
{
"key": "g",
"value": 0.0,
"min_value": -1,
"max_value": 1,
"_id": "19161646843249846"
},
{
"key": "b",
"value": 0.0,
"min_value": -1,
"max_value": 1,
"_id": "12+92+17+174+9174+5"
}
]
},
"Sin(xyz)": {
"edit_text": "gs._xyz = torch.sin(gs._xyz/3+slider.x) + gs._xyz\n",
"slider": [
{
"key": "x",
"value": 0.0,
"min_value": -10,
"max_value": 10,
"_id": "28971294217951247"
}
]
},
"Random Colors": {
"edit_text": "torch.manual_seed(0)\ngs._features_dc = torch.rand_like(gs._features_dc) * slider.intensity - slider.intensity / 2\ngs._features_rest = gs._features_rest * 0\n\n",
"slider": [
{
"key": "intensity",
"value": 5.0,
"min_value": 0,
"max_value": 10,
"_id": "9f3fd0c1-67a5-4b38-9ec4-36d27aceddda"
}
]
},
"Recoloring": {
"edit_text": "selected_color = torch.tensor([slider.r, slider.g, slider.b], device=\"cuda\")\n\ngs._xyz = gs._xyz\ngs._rotation = gs._rotation\ngs._scaling = gs._scaling\ngs._opacity = gs._opacity\ngs._features_dc = gs._features_dc + selected_color\ngs._features_rest = gs._features_rest\n",
"slider": [
{
"key": "r",
"value": 0.0,
"min_value": -10,
"max_value": 10,
"_id": "9419751979747546"
},
{
"key": "g",
"value": 0.0,
"min_value": -10,
"max_value": 10,
"_id": "52795+7+5+4959597"
},
{
"key": "b",
"value": 0.0,
"min_value": -10,
"max_value": 10,
"_id": "47954795627648945"
}
]
},
"Filter Small gs": {
"edit_text": "mask = torch.linalg.norm(gs._scaling, dim=-1) < slider.scaling\n\ngs._xyz = gs._xyz[mask]\ngs._rotation = gs._rotation[mask]\ngs._scaling = gs._scaling[mask]\ngs._opacity = gs._opacity[mask]\ngs._features_dc = gs._features_dc[mask]\ngs._features_rest = gs._features_rest[mask]\n",
"slider": [
{
"key": "scaling",
"value": 10.0,
"min_value": 0,
"max_value": 20,
"_id": "8769fe63-c073-4b0f-b83d-dc519024a0bb"
}
]
},
"Wobbel": {
"edit_text": "gs._xyz[:,2] = torch.sin(gs._xyz[:,2]+slider.x)/5 + gs._xyz[:,2]\ngs._xyz[:,0] = torch.sin(gs._xyz[:,0]+slider.x)/5 + gs._xyz[:,0]",
"slider": [
{
"key": "x",
"value": -10.0,
"min_value": -10.0,
"max_value": 10.0,
"_id": "28971294217951247"
}
]
},
"Slice Scene": {
"edit_text": "mask_x = gs._xyz[:, 0] < slider.x\nmask_y =gs._xyz[:, 1] < slider.y\nmask_z =gs._xyz[:, 2] < slider.z\n\nmask = mask_x & mask_y & mask_z\n\ngs._xyz = gs._xyz[mask]\ngs._rotation = gs._rotation[mask]\ngs._scaling = gs._scaling[mask]\ngs._opacity = gs._opacity[mask]\ngs._features_dc = gs._features_dc[mask]\ngs._features_rest = gs._features_rest[mask]\n",
"slider": [
{
"key": "x",
"value": -0.0010000000474974513,
"min_value": -5.0,
"max_value": 5.0,
"_id": "12364012634"
},
{
"key": "y",
"value": 0.0,
"min_value": -5.0,
"max_value": 5.0,
"_id": "197e4266-82ba-417d-b276-536e465f4fc0"
},
{
"key": "z",
"value": -0.0,
"min_value": -5.0,
"max_value": 5.0,
"_id": "636299d8-5d63-4e7c-8ce4-d192aa019d95"
}
]
},
"Deconstructing Self-Organizing Gaussians": {
"edit_text": "num_gaussians = gs._xyz.shape[0]\ngrid_sidelen = int(np.sqrt(num_gaussians))\ngrid_ids = torch.arange(num_gaussians, device=gs._xyz.device)\n\ngrid_us = (grid_ids % grid_sidelen) / grid_sidelen - 0.5\ngrid_vs = grid_ids // grid_sidelen / grid_sidelen - 0.5\n\ngrid_xyz = torch.zeros((num_gaussians, 3), device=gs._xyz.device)\n\ngrid_xyz[:, 1] = grid_us * slider.size\ngrid_xyz[:, 2] = grid_vs.flip(dims=(0,)) * slider.size\n\ngs.grid_ids = grid_ids\ngs._xyz = (1.0 - slider.phase) * gs._xyz + slider.phase * grid_xyz\ngs._rotation = gs._rotation\ngs._scaling = (1.0 - slider.phase) * gs._scaling + slider.phase * slider.grid_scale\ngs._opacity = (1.0 - slider.phase) * gs._opacity + slider.phase * slider.grid_opacity\ngs._features_dc = gs._features_dc\ngs._features_rest = gs._features_rest\n",
"slider": [
{
"key": "phase",
"value": 1.0,
"min_value": 0.0,
"max_value": 1.0,
"animating": false,
"loop_animation": true,
"animation_duration_sec": 5,
"_id": "a8f2a853-17ba-4f7f-9dd8-843d2f0756a1"
},
{
"key": "size",
"value": 1.5,
"min_value": 0.5,
"max_value": 4.0,
"animating": false,
"loop_animation": true,
"animation_duration_sec": 5,
"_id": "f6d7db6e-d8a2-4513-9317-e73f7fe3328b"
},
{
"key": "grid_scale",
"value": -8.0,
"min_value": -8.0,
"max_value": -4.0,
"animating": false,
"loop_animation": true,
"animation_duration_sec": 5,
"_id": "38cc51e9-7018-45a6-968e-075ae68f9bfe"
},
{
"key": "grid_opacity",
"value": 0,
"min_value": -4.0,
"max_value": 1.0,
"animating": false,
"loop_animation": true,
"animation_duration_sec": 5,
"_id": "7c80d17e-c081-4743-90b6-3058afbba4b7"
}
]
}
}
39 changes: 29 additions & 10 deletions splatviz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,18 @@
from splatviz_utils.gui_utils.constants import *
from splatviz_utils.dict_utils import EasyDict
from widgets import (
camera_circle_widget,
edit_widget,
eval_widget,
performance_widget,
load_widget_pkl,
load_widget_ply,
video_widget,
cam_widget,
capture_widget,
latent_widget,
render_widget,
training_widget,
video_recording_widget,
)


Expand All @@ -52,26 +53,26 @@ def __init__(self, data_path, mode, host, port, ggd_path=""):
# Internals.
self._last_error_print = None

self.widgets = []
pre_render_widgets = []
update_all_the_time = False
if mode == "default":
self.widgets = [
pre_render_widgets = [
load_widget_ply.LoadWidget(self, data_path),
cam_widget.CamWidget(self),
performance_widget.PerformanceWidget(self),
video_widget.VideoWidget(self),
camera_circle_widget.CameraCircleWidget(self),
capture_widget.CaptureWidget(self),
render_widget.RenderWidget(self),
edit_widget.EditWidget(self),
eval_widget.EvalWidget(self),
]
renderer = GaussianRenderer()
elif mode == "decoder":
self.widgets = [
pre_render_widgets = [
load_widget_pkl.LoadWidget(self, data_path),
cam_widget.CamWidget(self),
performance_widget.PerformanceWidget(self),
video_widget.VideoWidget(self),
camera_circle_widget.CameraCircleWidget(self),
capture_widget.CaptureWidget(self),
render_widget.RenderWidget(self),
edit_widget.EditWidget(self),
Expand All @@ -84,10 +85,10 @@ def __init__(self, data_path, mode, host, port, ggd_path=""):
sys.path.append(ggd_path + "/main")
renderer = GaussianDecoderRenderer()
elif mode == "attach":
self.widgets = [
pre_render_widgets = [
cam_widget.CamWidget(self),
performance_widget.PerformanceWidget(self),
video_widget.VideoWidget(self),
camera_circle_widget.CameraCircleWidget(self),
render_widget.RenderWidget(self),
edit_widget.EditWidget(self),
training_widget.TrainingWidget(self),
Expand All @@ -97,6 +98,12 @@ def __init__(self, data_path, mode, host, port, ggd_path=""):
else:
raise NotImplementedError(f"Mode '{mode}' not recognized.")

self.pre_render_widgets = pre_render_widgets

self.post_render_widgets = [
video_recording_widget.VideoRecordingWidget(self),
]

self.renderer = RendererWrapper(renderer, update_all_the_time)
self._tex_img = None
self._tex_obj = None
Expand All @@ -112,8 +119,12 @@ def __init__(self, data_path, mode, host, port, ggd_path=""):
self.skip_frame()

def close(self):
for widget in self.widgets:
for widget in self.pre_render_widgets:
widget.close()

for widget in self.post_render_widgets:
widget.close()

super().close()

def print_error(self, error):
Expand Down Expand Up @@ -147,7 +158,7 @@ def draw_frame(self):
imgui.begin("##control_pane", p_open=True, flags=control_pane_flags)

# Widgets
for widget in self.widgets:
for widget in self.pre_render_widgets:
expanded, _visible = imgui_utils.collapsing_header(widget.name, default=widget.name == "Load")
imgui.indent()
widget(expanded)
Expand All @@ -164,6 +175,14 @@ def draw_frame(self):
if result is not None:
self.result = result

# Widgets
for widget in self.post_render_widgets:
expanded, _visible = imgui_utils.collapsing_header(widget.name, default=widget.name == "Load")
imgui.indent()
widget(expanded)
imgui.unindent()


# Display
max_w = self.content_width - self.pane_w
max_h = self.content_height
Expand Down
4 changes: 2 additions & 2 deletions widgets/video_widget.py → widgets/camera_circle_widget.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
from scene.cameras import CustomCam


class VideoWidget(Widget):
class CameraCircleWidget(Widget):
def __init__(self, viz):
super().__init__(viz, "Video")
super().__init__(viz, "Camera Circle")
self.num_frames = 1000
self.cam_height = 0.3
self.radius = 6
Expand Down
Loading