No checkpoints to load, training from scratch
nerfstudio field components: Setting up CUDA (This may take a few minutes the first time)
Printing profiling stats, from longest to shortest duration in seconds
Traceback (most recent call last):
  File "/home/user/.local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1808, in _run_ninja_build
    subprocess.run(
  File "/usr/lib/python3.10/subprocess.py", line 524, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/user/.local/bin/ns-train", line 8, in <module>
    sys.exit(entrypoint())
  File "/home/user/nerfstudio/scripts/train.py", line 247, in entrypoint
    main(
  File "/home/user/nerfstudio/scripts/train.py", line 233, in main
    launch(
  File "/home/user/nerfstudio/scripts/train.py", line 172, in launch
    main_func(local_rank=0, world_size=world_size, config=config)
  File "/home/user/nerfstudio/scripts/train.py", line 87, in train_loop
    trainer.train()
  File "/home/user/nerfstudio/nerfstudio/engine/trainer.py", line 198, in train
    callback.run_callback_at_location(
  File "/home/user/nerfstudio/nerfstudio/engine/callbacks.py", line 103, in run_callback_at_location
    self.run_callback(step=step)
  File "/home/user/nerfstudio/nerfstudio/engine/callbacks.py", line 90, in run_callback
    self.func(*self.args, **self.kwargs, step=step)
  File "/home/user/nerfstudio/nerfstudio/models/instant_ngp.py", line 148, in update_occupancy_grid
    self.occupancy_grid.every_n_step(
  File "/home/user/.local/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/home/user/.local/lib/python3.10/site-packages/nerfacc/grid.py", line 271, in every_n_step
    self._update(
  File "/home/user/.local/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/home/user/.local/lib/python3.10/site-packages/nerfacc/grid.py", line 229, in _update
    occ = occ_eval_fn(x).squeeze(-1)
  File "/home/user/nerfstudio/nerfstudio/models/instant_ngp.py", line 150, in <lambda>
    occ_eval_fn=lambda x: self.field.get_opacity(x, self.config.render_step_size),
  File "/home/user/nerfstudio/nerfstudio/fields/nerfplayer_ngp_field.py", line 233, in get_opacity
    density = self.density_fn(positions, t / (time_intervals - 1) * torch.ones_like(positions)[..., [0]])
  File "/home/user/nerfstudio/nerfstudio/fields/nerfplayer_ngp_field.py", line 217, in density_fn
    density, _ = self.get_density(ray_samples)
  File "/home/user/nerfstudio/nerfstudio/fields/nerfplayer_ngp_field.py", line 154, in get_density
    h = self.mlp_base(positions_flat, times_flat)
  File "/home/user/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/user/nerfstudio/nerfstudio/field_components/temporal_grid.py", line 336, in forward
    outputs = TemporalGridEncodeFunc.apply(
  File "/home/user/.local/lib/python3.10/site-packages/torch/cuda/amp/autocast_mode.py", line 110, in decorate_fwd
    return fwd(*args, **kwargs)
  File "/home/user/nerfstudio/nerfstudio/field_components/temporal_grid.py", line 89, in forward
    _C.temporal_grid_encode_forward(
  File "/home/user/nerfstudio/nerfstudio/field_components/cuda/__init__.py", line 24, in call_cuda
    from ._backend import _C
  File "/home/user/nerfstudio/nerfstudio/field_components/cuda/_backend.py", line 46, in <module>
    _C = load(
  File "/home/user/.local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1202, in load
    return _jit_compile(
  File "/home/user/.local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1425, in _jit_compile
    _write_ninja_file_and_build_library(
  File "/home/user/.local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1537, in _write_ninja_file_and_build_library
    _run_ninja_build(
  File "/home/user/.local/lib/python3.10/site-packages/torch/utils/cpp_extension.py", line 1824, in _run_ninja_build
    raise RuntimeError(message) from e
RuntimeError: Error building extension 'nerfstudio_field_components_cuda': [1/3] /usr/local/cuda/bin/nvcc  -DTORCH_EXTENSION_NAME=nerfstudio_field_components_cuda -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /home/user/.local/lib/python3.10/site-packages/torch/include -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/TH -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /usr/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_61,code=compute_61 -gencode=arch=compute_61,code=sm_61 --compiler-options '-fPIC' -O3 -std=c++14 -c /home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu -o temporal_gridencoder.cuda.o
FAILED: temporal_gridencoder.cuda.o
/usr/local/cuda/bin/nvcc  -DTORCH_EXTENSION_NAME=nerfstudio_field_components_cuda -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /home/user/.local/lib/python3.10/site-packages/torch/include -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/TH -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /usr/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_61,code=compute_61 -gencode=arch=compute_61,code=sm_61 --compiler-options '-fPIC' -O3 -std=c++14 -c /home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu -o temporal_gridencoder.cuda.o
/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(36): error: no instance of overloaded function "atomicAdd" matches the argument list
            argument types are: (__half *, c10::Half)

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U, C=1U]"
(425): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U, C=2U]"
(428): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U, C=4U]"
(431): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U, C=8U]"
(434): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=double]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U, C=1U]"
(425): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U, C=2U]"
(428): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U, C=4U]"
(431): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U, C=8U]"
(434): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=float]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U, C=1U]"
(425): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U, C=2U]"
(428): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U, C=4U]"
(431): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half]"
(610): here

/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu(222): warning #186-D: pointless comparison of unsigned integer with zero
          detected during:
            instantiation of "void kernel_grid<scalar_t,D,C>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U, C=8U]"
(434): here
            instantiation of "void kernel_grid_wrapper<scalar_t,D>(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half, D=1U]"
(460): here
            instantiation of "void temporal_grid_encode_forward_cuda(const float *, const float *, const scalar_t *, const int *, scalar_t *, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, float, uint32_t, scalar_t *, uint32_t, __nv_bool) [with scalar_t=c10::Half]"
(610): here

1 error detected in the compilation of "/home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/temporal_gridencoder.cu".
[2/3] /usr/local/cuda/bin/nvcc  -DTORCH_EXTENSION_NAME=nerfstudio_field_components_cuda -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -isystem /home/user/.local/lib/python3.10/site-packages/torch/include -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/torch/csrc/api/include -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/TH -isystem /home/user/.local/lib/python3.10/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /usr/include/python3.10 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_61,code=compute_61 -gencode=arch=compute_61,code=sm_61 --compiler-options '-fPIC' -O3 -std=c++14 -c /home/user/nerfstudio/nerfstudio/field_components/cuda/csrc/pybind.cu -o pybind.cuda.o
ninja: build stopped: subcommand failed.