PPO evaluation without the docker

[ghost]

❓ Questions and Help

Hi!

I trained PPO agent model from habitat_baselines folder on pointnav task with gibson dataset. I wanted to evaluate SPL of trained model without the docker. I modified habitat_api/examples/benchmark.py file as following:

 #!/usr/bin/env python3

import argparse

import habitat
from habitat.config import Config
from habitat.config.default import get_config
from habitat_baselines.agents.ppo_agents import PPOAgent
from habitat.sims.habitat_simulator.actions import HabitatSimActions


def get_default_config():
    c = Config()
    c.INPUT_TYPE = "rgbd" 
    c.MODEL_PATH = "data/new_checkpoints/ckpt.199.pth" #my trained ppo model
    c.RESOLUTION = 256
    c.HIDDEN_SIZE = 512
    c.RANDOM_SEED = 7
    c.PTH_GPU_ID = 0
    c.GOAL_SENSOR_UUID = "pointgoal_with_gps_compass"
    return c


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--task-config", type=str, default="habitat_baselines/config/pointnav/ppo_pointnav.yaml"
    )
    args = parser.parse_args()

    config = get_config(args.task_config)
    agent_config = get_default_config()
    agent_config.GOAL_SENSOR_UUID = config.TASK.GOAL_SENSOR_UUID

    agent = PPOAgent(agent_config)
    benchmark = habitat.Benchmark(config_paths=args.task_config)
    metrics = benchmark.evaluate(agent)

    print(metrics.items())
    for k, v in metrics.items():
        print("{}: {:.3f}".format(k, v))


if __name__ == "__main__":
    main()

After running the script I got following error:

2020-03-10 10:58:13,311 Initializing dataset PointNav-v1
2020-03-10 10:58:13,526 initializing sim Sim-v0
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0310 10:58:13.533234  6152 SceneGraph.h:92] Created DrawableGroup: 
I0310 10:58:13.538457  6152 WindowlessContext.cpp:100] [EGL] Detected 5 EGL devices
I0310 10:58:13.538547  6152 WindowlessContext.cpp:121] [EGL] Selected EGL device 0 for CUDA device 0
I0310 10:58:13.541143  6152 WindowlessContext.cpp:135] [EGL] Version: 1.5
I0310 10:58:13.541152  6152 WindowlessContext.cpp:136] [EGL] Vendor: NVIDIA
I0310 10:58:13.573535  6152 ResourceManager.cpp:1026] Importing Basis files as BC7
W0310 10:58:14.301607  6152 Simulator.cpp:143] :
---
 The active scene does not contain semantic annotations. 
---
I0310 10:58:14.301897 6152 simulator.py:143] Loaded navmesh data/scene_datasets/habitat-test-scenes/van-gogh-room.navmesh
2020-03-10 10:58:14,303 Initializing task Nav-v0
Traceback (most recent call last):
  File "examples/benchmark_ppo.py", line 48, in <module>
    main()
  File "examples/benchmark_ppo.py", line 40, in main
    metrics = benchmark.evaluate(agent)
  File "/home/pryhoda/HabitatProject/habitat-api/habitat/core/benchmark.py", line 63, in evaluate
    action = agent.act(observations)
  File "/home/pryhoda/HabitatProject/habitat-api/habitat_baselines/agents/ppo_agents.py", line 134, in act
    deterministic=False,
  File "/home/pryhoda/HabitatProject/habitat-api/habitat_baselines/rl/ppo/policy.py", line 40, in act
    observations, rnn_hidden_states, prev_actions, masks
  File "/home/pryhoda/anaconda3/envs/habitat/lib/python3.6/site-packages/torch-1.4.0-py3.6-linux-x86_64.egg/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/pryhoda/HabitatProject/habitat-api/habitat_baselines/rl/ppo/policy.py", line 167, in forward
    perception_embed = self.visual_encoder(observations)
  File "/home/pryhoda/anaconda3/envs/habitat/lib/python3.6/site-packages/torch-1.4.0-py3.6-linux-x86_64.egg/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/pryhoda/HabitatProject/habitat-api/habitat_baselines/rl/models/simple_cnn.py", line 142, in forward
    depth_observations = depth_observations.permute(0, 3, 1, 2)
AttributeError: 'list' object has no attribute 'permute'
I0310 10:58:14.448240  6152 Simulator.cpp:36] Deconstructing Simulator
I0310 10:58:14.448280  6152 SemanticScene.h:40] Deconstructing SemanticScene
I0310 10:58:14.448284  6152 SceneManager.h:24] Deconstructing SceneManager
I0310 10:58:14.448287  6152 SceneGraph.h:25] Deconstructing SceneGraph
I0310 10:58:14.448312  6152 RenderTarget.h:51] Deconstructing RenderTarget
I0310 10:58:14.448774  6152 Sensor.h:80] Deconstructing Sensor
I0310 10:58:14.448931  6152 Renderer.cpp:33] Deconstructing Renderer
I0310 10:58:14.448945  6152 WindowlessContext.h:16] Deconstructing WindowlessContext
I0310 10:58:14.448947  6152 WindowlessContext.cpp:245] Deconstructing GL context

I am wondering if I am loading some wrong configurations or it is some internal habitat bug ?