{"id":189,"date":"2026-05-07T22:54:43","date_gmt":"2026-05-07T22:54:43","guid":{"rendered":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/?page_id=189"},"modified":"2026-05-08T05:06:39","modified_gmt":"2026-05-08T05:06:39","slug":"method","status":"publish","type":"page","link":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/","title":{"rendered":"Method"},"content":{"rendered":"\n<div class=\"wp-block-group alignfull has-global-padding is-layout-constrained wp-block-group-is-layout-constrained\">\n<h2 class=\"wp-block-heading\">4D Reconstruction From Monocular Video<\/h2>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"416\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-1024x416.jpg\" alt=\"\" class=\"wp-image-144\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-1024x416.jpg 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-300x122.jpg 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-768x312.jpg 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-1536x625.jpg 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-2048x833.jpg 2048w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p><strong>Authors<\/strong>: <span style=\"text-decoration: underline\">Manan Shah<\/span>. <strong>Project Advisors<\/strong>: Yehonathan Litman, Xiaoxuan Ma, Nicolas Ugrinovic, Kris Kitani, Fernando De La Torre, Shubham Tulsiani<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\" \/>\n\n\n\n<div class=\"wp-block-group has-global-padding is-layout-constrained wp-block-group-is-layout-constrained\"><nav class=\"is-responsive wp-block-navigation is-layout-flex wp-block-navigation-is-layout-flex\" aria-label=\"Menu\" \n\t\t data-wp-interactive=\"core\/navigation\" data-wp-context='{\"overlayOpenedBy\":{\"click\":false,\"hover\":false,\"focus\":false},\"type\":\"overlay\",\"roleAttribute\":\"\",\"ariaLabel\":\"Menu\"}'><button aria-haspopup=\"dialog\" aria-label=\"Open menu\" class=\"wp-block-navigation__responsive-container-open\" \n\t\t\t\tdata-wp-on--click=\"actions.openMenuOnClick\"\n\t\t\t\tdata-wp-on--keydown=\"actions.handleMenuKeydown\"\n\t\t\t><svg width=\"24\" height=\"24\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 24 24\" aria-hidden=\"true\" focusable=\"false\"><path d=\"M4 7.5h16v1.5H4z\"><\/path><path d=\"M4 15h16v1.5H4z\"><\/path><\/svg><\/button>\n\t\t\t\t<div class=\"wp-block-navigation__responsive-container\"  id=\"modal-1\" \n\t\t\t\tdata-wp-class--has-modal-open=\"state.isMenuOpen\"\n\t\t\t\tdata-wp-class--is-menu-open=\"state.isMenuOpen\"\n\t\t\t\tdata-wp-watch=\"callbacks.initMenu\"\n\t\t\t\tdata-wp-on--keydown=\"actions.handleMenuKeydown\"\n\t\t\t\tdata-wp-on--focusout=\"actions.handleMenuFocusout\"\n\t\t\t\ttabindex=\"-1\"\n\t\t\t>\n\t\t\t\t\t<div class=\"wp-block-navigation__responsive-close\" tabindex=\"-1\">\n\t\t\t\t\t\t<div class=\"wp-block-navigation__responsive-dialog\" \n\t\t\t\tdata-wp-bind--aria-modal=\"state.ariaModal\"\n\t\t\t\tdata-wp-bind--aria-label=\"state.ariaLabel\"\n\t\t\t\tdata-wp-bind--role=\"state.roleAttribute\"\n\t\t\t>\n\t\t\t\t\t\t\t<button aria-label=\"Close menu\" class=\"wp-block-navigation__responsive-container-close\" \n\t\t\t\tdata-wp-on--click=\"actions.closeMenuOnClick\"\n\t\t\t><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 24 24\" width=\"24\" height=\"24\" aria-hidden=\"true\" focusable=\"false\"><path d=\"m13.06 12 6.47-6.47-1.06-1.06L12 10.94 5.53 4.47 4.47 5.53 10.94 12l-6.47 6.47 1.06 1.06L12 13.06l6.47 6.47 1.06-1.06L13.06 12Z\"><\/path><\/svg><\/button>\n\t\t\t\t\t\t\t<div class=\"wp-block-navigation__responsive-container-content\" \n\t\t\t\tdata-wp-watch=\"callbacks.focusFirstElement\"\n\t\t\t id=\"modal-1-content\">\n\t\t\t\t\t\t\t\t<ul class=\"wp-block-navigation__container is-responsive wp-block-navigation\"><li class=\" wp-block-navigation-item wp-block-navigation-link\"><a class=\"wp-block-navigation-item__content\"  href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/introduction\/\"><span class=\"wp-block-navigation-item__label\">Introduction<\/span><\/a><\/li><li class=\" wp-block-navigation-item wp-block-navigation-link\"><a class=\"wp-block-navigation-item__content\"  href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/related-works\/\"><span class=\"wp-block-navigation-item__label\">Related Works<\/span><\/a><\/li><li class=\" wp-block-navigation-item wp-block-navigation-link\"><a class=\"wp-block-navigation-item__content\"  href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/\"><span class=\"wp-block-navigation-item__label\">Method<\/span><\/a><\/li><li class=\" wp-block-navigation-item wp-block-navigation-link\"><a class=\"wp-block-navigation-item__content\"  href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/results\/\"><span class=\"wp-block-navigation-item__label\">Results<\/span><\/a><\/li><li class=\" wp-block-navigation-item wp-block-navigation-link\"><a class=\"wp-block-navigation-item__content\"  href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/resources\/\"><span class=\"wp-block-navigation-item__label\">Resources<\/span><\/a><\/li><li class=\" wp-block-navigation-item wp-block-navigation-link\"><a class=\"wp-block-navigation-item__content\"  href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/team\/\"><span class=\"wp-block-navigation-item__label\">Team<\/span><\/a><\/li><\/ul>\n\t\t\t\t\t\t\t<\/div>\n\t\t\t\t\t\t<\/div>\n\t\t\t\t\t<\/div>\n\t\t\t\t<\/div><\/nav><\/div>\n<\/div>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\" \/>\n\n\n\n<h1 class=\"wp-block-heading\" id=\"introduction\">Method<\/h1>\n\n\n\n<p>Given a monocular video sequence <math data-latex=\"\\mathcal{I} = \\{\\mathbf{I}^i\\}_{i=1}^N\"><semantics><mrow><mi class=\"mathcal\">\u2110<\/mi><mo>=<\/mo><mo form=\"prefix\" stretchy=\"false\">{<\/mo><msup><mi>\ud835\udc08<\/mi><mi>i<\/mi><\/msup><msubsup><mo form=\"postfix\" stretchy=\"false\">}<\/mo><mrow><mi>i<\/mi><mo>=<\/mo><mn>1<\/mn><\/mrow><mi>N<\/mi><\/msubsup><\/mrow><annotation encoding=\"application\/x-tex\">\\mathcal{I} = \\{\\mathbf{I}^i\\}_{i=1}^N<\/annotation><\/semantics><\/math> with associated object masks <math data-latex=\"\\mathcal{M} = \\{\\mathbf{M}^i\\}_{i=1}^N\"><semantics><mrow><mi class=\"mathcal\">\u2133<\/mi><mo>=<\/mo><mo form=\"prefix\" stretchy=\"false\">{<\/mo><msup><mi>\ud835\udc0c<\/mi><mi>i<\/mi><\/msup><msubsup><mo form=\"postfix\" stretchy=\"false\">}<\/mo><mrow><mi>i<\/mi><mo>=<\/mo><mn>1<\/mn><\/mrow><mi>N<\/mi><\/msubsup><\/mrow><annotation encoding=\"application\/x-tex\">\\mathcal{M} = \\{\\mathbf{M}^i\\}_{i=1}^N<\/annotation><\/semantics><\/math>, our goal is to reconstruct a 4D representation, which is factorized into a canonical set of <math data-latex=\"N_\\mathcal{G}\"><semantics><msub><mi>N<\/mi><mi class=\"mathcal\">\ud835\udca2<\/mi><\/msub><annotation encoding=\"application\/x-tex\">N_\\mathcal{G}<\/annotation><\/semantics><\/math> 3D gaussians <math data-latex=\"\\mathcal{G}^\\star = \\{\\boldsymbol{g}_m^\\star\\}_{m=1}^{N_\\mathcal{G}}\"><semantics><mrow><msup><mi class=\"mathcal\">\ud835\udca2<\/mi><mo>\u22c6<\/mo><\/msup><mo>=<\/mo><mo form=\"prefix\" stretchy=\"false\">{<\/mo><msubsup><mi>\ud835\udc88<\/mi><mi>m<\/mi><mo form=\"prefix\" stretchy=\"false\">\u22c6<\/mo><\/msubsup><msubsup><mo form=\"postfix\" stretchy=\"false\">}<\/mo><mrow><mi>m<\/mi><mo>=<\/mo><mn>1<\/mn><\/mrow><msub><mi>N<\/mi><mi class=\"mathcal\">\ud835\udca2<\/mi><\/msub><\/msubsup><\/mrow><annotation encoding=\"application\/x-tex\">\\mathcal{G}^\\star = \\{\\boldsymbol{g}_m^\\star\\}_{m=1}^{N_\\mathcal{G}}<\/annotation><\/semantics><\/math> and a sequence of per-frame deformation fields <math data-latex=\"\\{D^i\\}_{i=1}^N\"><semantics><mrow><mo form=\"prefix\" stretchy=\"false\">{<\/mo><msup><mi>D<\/mi><mi>i<\/mi><\/msup><msubsup><mo form=\"postfix\" stretchy=\"false\">}<\/mo><mrow><mi>i<\/mi><mo>=<\/mo><mn>1<\/mn><\/mrow><mi>N<\/mi><\/msubsup><\/mrow><annotation encoding=\"application\/x-tex\">\\{D^i\\}_{i=1}^N<\/annotation><\/semantics><\/math>. To efficiently represent the deformation, we parameterize <math data-latex=\"D^i\"><semantics><msup><mi>D<\/mi><mi>i<\/mi><\/msup><annotation encoding=\"application\/x-tex\">D^i<\/annotation><\/semantics><\/math> through a sparse set of <math data-latex=\"N_p\"><semantics><msub><mi>N<\/mi><mi>p<\/mi><\/msub><annotation encoding=\"application\/x-tex\">N_p<\/annotation><\/semantics><\/math> control nodes to deform each canonical gaussian with time-varying SE(3) transformations.To achieve this, our framework consists of three modules. First, we initialize a sequence of 3D gaussian reconstructions using an off-the-shelf image-to-3D model, while enforcing temporal consistency across frames through <em>causal latent conditioning<\/em>. Next, we use these per-frame single-view reconstructions to guide the learning of our 4D representation, where both canonical gaussians and per-frame deformation fields are jointly optimized to build a coherent deformable 3D structure. Finally, we match the reconstructed representation to the input video through occlusion-aware rendering while also incorporating generic image priors for plausible reconstruction in unobserved regions.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Causal Single-view Reconstruction<\/h2>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"278\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/causal-1-1024x278.png\" alt=\"\" class=\"wp-image-199\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/causal-1-1024x278.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/causal-1-300x81.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/causal-1-768x209.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/causal-1-1536x417.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/causal-1.png 1930w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><figcaption class=\"wp-element-caption\"><strong>Causal latent conditioning: <\/strong>Given a video input, we obtain per-frame 3D reconstructions with SAM3D using causal latent conditioning to enforce the temporal consistency across frames.<\/figcaption><\/figure>\n\n\n\n<p>We begin by obtaining a per-frame 3D reconstruction using an off-the-shelf flow matching-based image-to-3D model (SAM3D<sup data-fn=\"0bef2719-45ae-4122-8035-fd16db057b10\" class=\"fn\"><a href=\"#0bef2719-45ae-4122-8035-fd16db057b10\" id=\"0bef2719-45ae-4122-8035-fd16db057b10-link\">1<\/a><\/sup>). While such a model can reconstruct plausible 3D gaussians from a single image, applying them independently to each frame of a video leads to temporal inconsistencies. To alleviate this issue, we enforce temporal consistency through causal latent propagation as shown in the figure.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Deformable 3D Representation and Optimization<\/h2>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"241\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/SC-GS-1-1024x241.png\" alt=\"\" class=\"wp-image-200\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/SC-GS-1-1024x241.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/SC-GS-1-300x71.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/SC-GS-1-768x181.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/SC-GS-1-1536x362.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/SC-GS-1.png 1952w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p>We factorize the 4D representation into canonical 3D gaussians <math data-latex=\"\\mathcal{G}^\\star\"><semantics><msup><mi class=\"mathcal\">\ud835\udca2<\/mi><mo>\u22c6<\/mo><\/msup><annotation encoding=\"application\/x-tex\">\\mathcal{G}^\\star<\/annotation><\/semantics><\/math>and per-frame deformation fields <math data-latex=\"\\{D^i\\}_{i=1}^N\"><semantics><mrow><mo form=\"prefix\" stretchy=\"false\">{<\/mo><msup><mi>D<\/mi><mi>i<\/mi><\/msup><msubsup><mo form=\"postfix\" stretchy=\"false\">}<\/mo><mrow><mi>i<\/mi><mo>=<\/mo><mn>1<\/mn><\/mrow><mi>N<\/mi><\/msubsup><\/mrow><annotation encoding=\"application\/x-tex\">\\{D^i\\}_{i=1}^N<\/annotation><\/semantics><\/math> parameterized by a sparse set of control nodes similar to SC-GS<sup data-fn=\"3e6bd12c-e7c1-4199-a18d-96fb0e18e9ef\" class=\"fn\"><a href=\"#3e6bd12c-e7c1-4199-a18d-96fb0e18e9ef\" id=\"3e6bd12c-e7c1-4199-a18d-96fb0e18e9ef-link\">2<\/a><\/sup>. Each node is associated with a time-varying learnable transformation <math data-latex=\"[\\mathbf{R}^i_k|\\mathbf{t}^i_k]\\in \\mathrm{SE}(3)\"><semantics><mrow><mo form=\"prefix\" stretchy=\"false\">[<\/mo><msubsup><mi>\ud835\udc11<\/mi><mi>k<\/mi><mi>i<\/mi><\/msubsup><mi>|<\/mi><msubsup><mi>\ud835\udc2d<\/mi><mi>k<\/mi><mi>i<\/mi><\/msubsup><mo form=\"postfix\" stretchy=\"false\">]<\/mo><mo>\u2208<\/mo><mrow><mtext><\/mtext><mi>SE<\/mi><\/mrow><mo form=\"prefix\" stretchy=\"false\">(<\/mo><mn>3<\/mn><mo form=\"postfix\" stretchy=\"false\">)<\/mo><\/mrow><annotation encoding=\"application\/x-tex\">[\\mathbf{R}^i_k|\\mathbf{t}^i_k]\\in \\mathrm{SE}(3)<\/annotation><\/semantics><\/math>. The per-frame initialized 3D gaussians obtained from SAM3D then guide the joint optimization of the canonical gaussians and the deformation field.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Optimization Objective<\/h2>\n\n\n\n<p>The overall optimization objective to guide the canonical gaussians and deformation field is given by <math data-latex=\"\\mathcal{L}_\\text{rec}\"><semantics><msub><mi class=\"mathcal\">\u2112<\/mi><mtext>rec<\/mtext><\/msub><annotation encoding=\"application\/x-tex\">\\mathcal{L}_\\text{rec}<\/annotation><\/semantics><\/math>, which is further decomposed as follows,<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"619\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/optimization-objective-1-1024x619.png\" alt=\"\" class=\"wp-image-201\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/optimization-objective-1-1024x619.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/optimization-objective-1-300x181.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/optimization-objective-1-768x464.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/optimization-objective-1-1536x928.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/optimization-objective-1-2048x1238.png 2048w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<h2 class=\"wp-block-heading\">Occlusion-aware Rendering Supervision and Reconstruction<\/h2>\n\n\n\n<p>While the deformable 3D optimization provides a temporally coherent 4D representation across frames, it does not directly enforce alignment with the appearance observed in the input video. Moreover, complex occlusions and incomplete observations commonly present in in-the-wild videos make it difficult to rely solely on simple rendering supervision.<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"189\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-reconstruction-mask-1-1024x189.png\" alt=\"\" class=\"wp-image-204\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-reconstruction-mask-1-1024x189.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-reconstruction-mask-1-300x55.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-reconstruction-mask-1-768x142.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-reconstruction-mask-1-1536x283.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-reconstruction-mask-1-2048x378.png 2048w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><figcaption class=\"wp-element-caption\"><strong>Occlusion-aware Rendering Supervision<\/strong><\/figcaption><\/figure>\n\n\n\n<p>To address this, we introduce an occlusion-aware appearance reconstruction framework that preserves observed image details while maintaining plausible content in unseen regions. The occlusion mask and ground truth reference are constructed as follows:<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"54\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-equation-1024x54.png\" alt=\"\" class=\"wp-image-205\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-equation-1024x54.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-equation-300x16.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-equation-768x41.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-equation-1536x81.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-equation.png 1964w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<p>Using the above method to generate the ground truth, we use the following loss for optimization at this stage:<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"99\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-sds-loss-1024x99.png\" alt=\"\" class=\"wp-image-206\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-sds-loss-1024x99.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-sds-loss-300x29.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-sds-loss-768x74.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-sds-loss-1536x149.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-sds-loss.png 1964w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"231\" src=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-1-1024x231.png\" alt=\"\" class=\"wp-image-202\" srcset=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-1-1024x231.png 1024w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-1-300x68.png 300w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-1-768x173.png 768w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-1-1536x346.png 1536w, https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/occ-aware-1.png 1952w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><figcaption class=\"wp-element-caption\">We use the completed reference image to serve as the occlusion-aware rendering supervision to guide the reconstruction of occluded and unobserved regions.<\/figcaption><\/figure>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\" \/>\n\n\n\n<p><strong>References<\/strong><\/p>\n\n\n<ol class=\"wp-block-footnotes\"><li id=\"0bef2719-45ae-4122-8035-fd16db057b10\">Team, S.D., Chen, X., Chu, F.J., Gleize, P., Liang, K.J., Sax, A., Tang, H., Wang, W., Guo, M., Hardin, T., Li, X., Lin, A., Liu, J., Ma, Z., Sagar, A., Song, B., Wang, X., Yang, J., Zhang, B., Doll\u00e1r, P., Gkioxari, G., Feiszli, M., Malik, J.: Sam 3d: 3dfy anything in images. arXiv (2025) <a href=\"#0bef2719-45ae-4122-8035-fd16db057b10-link\" aria-label=\"Jump to footnote reference 1\">\u21a9\ufe0e<\/a><\/li><li id=\"3e6bd12c-e7c1-4199-a18d-96fb0e18e9ef\">Huang, Y.H., Sun, Y.T., Yang, Z., Lyu, X., Cao, Y.P., Qi, X.: Sc-gs: Sparsecontrolled gaussian splatting for editable dynamic scenes. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (2024) <a href=\"#3e6bd12c-e7c1-4199-a18d-96fb0e18e9ef-link\" aria-label=\"Jump to footnote reference 2\">\u21a9\ufe0e<\/a><\/li><\/ol>","protected":false},"excerpt":{"rendered":"<p>4D Reconstruction From Monocular Video Authors: Manan Shah. Project Advisors: Yehonathan Litman, Xiaoxuan Ma, Nicolas Ugrinovic, Kris Kitani, Fernando De La Torre, Shubham Tulsiani Method Given a monocular video sequence \u2110={\ud835\udc08i}i=1N\\mathcal{I} = \\{\\mathbf{I}^i\\}_{i=1}^N with associated object masks \u2133={\ud835\udc0ci}i=1N\\mathcal{M} = \\{\\mathbf{M}^i\\}_{i=1}^N, our goal is to reconstruct a 4D representation, which is factorized into a canonical set [&hellip;]<\/p>\n","protected":false},"author":305,"featured_media":0,"parent":104,"menu_order":0,"comment_status":"closed","ping_status":"closed","template":"page-no-title","meta":{"footnotes":"[{\"id\":\"0bef2719-45ae-4122-8035-fd16db057b10\",\"content\":\"Team, S.D., Chen, X., Chu, F.J., Gleize, P., Liang, K.J., Sax, A., Tang, H., Wang, W., Guo, M., Hardin, T., Li, X., Lin, A., Liu, J., Ma, Z., Sagar, A., Song, B., Wang, X., Yang, J., Zhang, B., Doll\\u00e1r, P., Gkioxari, G., Feiszli, M., Malik, J.: Sam 3d: 3dfy anything in images. arXiv (2025)\"},{\"id\":\"3e6bd12c-e7c1-4199-a18d-96fb0e18e9ef\",\"content\":\"Huang, Y.H., Sun, Y.T., Yang, Z., Lyu, X., Cao, Y.P., Qi, X.: Sc-gs: Sparsecontrolled gaussian splatting for editable dynamic scenes. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR) (2024)\"}]"},"class_list":["post-189","page","type-page","status-publish","hentry"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v27.4 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>Method - 3D Scene Understanding<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Method - 3D Scene Understanding\" \/>\n<meta property=\"og:description\" content=\"4D Reconstruction From Monocular Video Authors: Manan Shah. Project Advisors: Yehonathan Litman, Xiaoxuan Ma, Nicolas Ugrinovic, Kris Kitani, Fernando De La Torre, Shubham Tulsiani Method Given a monocular video sequence \u2110={\ud835\udc08i}i=1Nmathcal{I} = {mathbf{I}^i}_{i=1}^N with associated object masks \u2133={\ud835\udc0ci}i=1Nmathcal{M} = {mathbf{M}^i}_{i=1}^N, our goal is to reconstruct a 4D representation, which is factorized into a canonical set [&hellip;]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/\" \/>\n<meta property=\"og:site_name\" content=\"3D Scene Understanding\" \/>\n<meta property=\"article:modified_time\" content=\"2026-05-08T05:06:39+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1.jpg\" \/>\n\t<meta property=\"og:image:width\" content=\"2272\" \/>\n\t<meta property=\"og:image:height\" content=\"924\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/jpeg\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data1\" content=\"5 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/\",\"url\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/\",\"name\":\"Method - 3D Scene Understanding\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/wp-content\\\/uploads\\\/sites\\\/159\\\/2026\\\/05\\\/teaser_comp-1-1024x416.jpg\",\"datePublished\":\"2026-05-07T22:54:43+00:00\",\"dateModified\":\"2026-05-08T05:06:39+00:00\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/#primaryimage\",\"url\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/wp-content\\\/uploads\\\/sites\\\/159\\\/2026\\\/05\\\/teaser_comp-1.jpg\",\"contentUrl\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/wp-content\\\/uploads\\\/sites\\\/159\\\/2026\\\/05\\\/teaser_comp-1.jpg\",\"width\":2272,\"height\":924,\"caption\":\"Screenshot\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/method\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"4D Reconstruction From Monocular Video\",\"item\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/4d-reconstruction-from-monocular-video\\\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"Method\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/#website\",\"url\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/\",\"name\":\"3D Scene Understanding\",\"description\":\"Resources\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/mscvprojects.ri.cmu.edu\\\/2026teamf17\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Method - 3D Scene Understanding","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/","og_locale":"en_US","og_type":"article","og_title":"Method - 3D Scene Understanding","og_description":"4D Reconstruction From Monocular Video Authors: Manan Shah. Project Advisors: Yehonathan Litman, Xiaoxuan Ma, Nicolas Ugrinovic, Kris Kitani, Fernando De La Torre, Shubham Tulsiani Method Given a monocular video sequence \u2110={\ud835\udc08i}i=1Nmathcal{I} = {mathbf{I}^i}_{i=1}^N with associated object masks \u2133={\ud835\udc0ci}i=1Nmathcal{M} = {mathbf{M}^i}_{i=1}^N, our goal is to reconstruct a 4D representation, which is factorized into a canonical set [&hellip;]","og_url":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/","og_site_name":"3D Scene Understanding","article_modified_time":"2026-05-08T05:06:39+00:00","og_image":[{"width":2272,"height":924,"url":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1.jpg","type":"image\/jpeg"}],"twitter_card":"summary_large_image","twitter_misc":{"Est. reading time":"5 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/","url":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/","name":"Method - 3D Scene Understanding","isPartOf":{"@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/#website"},"primaryImageOfPage":{"@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/#primaryimage"},"image":{"@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/#primaryimage"},"thumbnailUrl":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1-1024x416.jpg","datePublished":"2026-05-07T22:54:43+00:00","dateModified":"2026-05-08T05:06:39+00:00","breadcrumb":{"@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/#primaryimage","url":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1.jpg","contentUrl":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-content\/uploads\/sites\/159\/2026\/05\/teaser_comp-1.jpg","width":2272,"height":924,"caption":"Screenshot"},{"@type":"BreadcrumbList","@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/method\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/"},{"@type":"ListItem","position":2,"name":"4D Reconstruction From Monocular Video","item":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/4d-reconstruction-from-monocular-video\/"},{"@type":"ListItem","position":3,"name":"Method"}]},{"@type":"WebSite","@id":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/#website","url":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/","name":"3D Scene Understanding","description":"Resources","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"}]}},"_links":{"self":[{"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/pages\/189","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/pages"}],"about":[{"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/types\/page"}],"author":[{"embeddable":true,"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/users\/305"}],"replies":[{"embeddable":true,"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/comments?post=189"}],"version-history":[{"count":5,"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/pages\/189\/revisions"}],"predecessor-version":[{"id":246,"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/pages\/189\/revisions\/246"}],"up":[{"embeddable":true,"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/pages\/104"}],"wp:attachment":[{"href":"https:\/\/mscvprojects.ri.cmu.edu\/2026teamf17\/wp-json\/wp\/v2\/media?parent=189"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}