I am having an error when launching this small script :

import torch
from layers.MatrixConvLSTM import MatrixConvLSTM

def test_dummy_events(batch_size=4, height=32, width=32, tmax=100000, num_event_max=512):

    #dummy batch
    events = torch.zeros((batch_size, num_event_max, 4), dtype=torch.int32)
    lengths = torch.randint(32, num_event_max, (batch_size,))
    events[...,0] = torch.randint(0, width, (batch_size, num_event_max))
    events[...,1] = torch.randint(0, height, (batch_size, num_event_max))
    events[...,2] = torch.randint(0, tmax, (batch_size, num_event_max))
    events[...,3] = torch.randint(0, 2, (batch_size, num_event_max))

    embedding_size = 0
    use_embedding = False
    matrix_lstm_type = "ConvLSTM"
    matrix_input_size = embedding_size + 1 if use_embedding else 1
    lstm_num_layers = 1
    input_shape = (None, None)
    matrix_input_size = 1
    matrix_hidden_size = 8
    matrix_region_stride = (1,1)
    matrix_region_shape = (1,1)  # receptive-field
    matrix_add_coords_feature = False
    matrix_add_time_feature_mode = "delay_norm"
    matrix_normalize_relative = True
    matrix_keep_most_recent = False
    matrix_frame_intervals = 1
    matrix_frame_intervals_mode = None

    MatrixLSTMClass = MatrixConvLSTM if matrix_lstm_type == "ConvLSTM" else MatrixLSTM
    layer = MatrixLSTMClass(input_shape,
                                      matrix_region_shape,
                                      matrix_region_stride, matrix_input_size,
                                      matrix_hidden_size, lstm_num_layers,
                                      bias=True, lstm_type=matrix_lstm_type,
                                      add_coords_feature=matrix_add_coords_feature,
                                      add_time_feature_mode=matrix_add_time_feature_mode,
                                      normalize_relative=matrix_normalize_relative,
                                      keep_most_recent=matrix_keep_most_recent,
                                      frame_intervals=matrix_frame_intervals,
                                      frame_intervals_mode=matrix_frame_intervals_mode)

    output_dense = layer(events, lengths)
    print(output_dense.shape)

if __name__ == '__main__':
    import fire
    fire.Fire(test_dummy_events)

Error:

Traceback (most recent call last):
  File "scripts/test_matrixlstm.py", line 73, in <module>
    fire.Fire(test_dummy_events)
  File "/usr/local/lib/python3.6/dist-packages/fire/core.py", line 138, in Fire
    component_trace = _Fire(component, args, parsed_flag_args, context, name)
  File "/usr/local/lib/python3.6/dist-packages/fire/core.py", line 471, in _Fire
    target=component.__name__)
  File "/usr/local/lib/python3.6/dist-packages/fire/core.py", line 675, in _CallAndUpdateTrace
    component = fn(*varargs, **kwargs)
  File "scripts/test_matrixlstm.py", line 60, in test_dummy_events
    frame_intervals_mode=matrix_frame_intervals_mode)
  File "/home/etienneperot/workspace/matrixlstm/classification/layers/MatrixConvLSTM.py", line 54, in
 __init__
    num_layers=self.num_layers, batch_first=False)
  File "/home/etienneperot/workspace/matrixlstm/classification/libs/lstms/convlstm.py", line 409, in
__init__
    forget_bias=self.forget_bias[i]))
  File "/home/etienneperot/workspace/matrixlstm/classification/libs/lstms/convlstm.py", line 117, in
__init__
    self.Wi_padding = (self.Wi_kernel - 1) // 2, (self.Wi_kernel - 1) // 2
TypeError: unsupported operand type(s) for -: 'tuple' and 'int'

Probably in "libs/conv_lstm.py" you want to change lines 116 & 118 to:

116 self.Wi_padding = (self.Wi_kernel[0] - 1) // 2, (self.Wi_kernel[1] - 1) // 2
118 self.Wh_padding = (self.Wh_kernel[0] - 1) // 2, (self.Wh_kernel[1] - 1) // 2

@marcocannici Thanks for your outstanding work.

I used conda to create a new environment on my computer. The code can run normally, but I found that the speed is slow. So I set to use multiple GPUs (0 ,1) in the parameters, but in the end only the last GPU is working.

I run the following command under the classification folder.

python scripts/train_matrixlstm_resnet_decay.py 0,1 -c configs/xxx.yaml

Maybe there is a problem with my parameter passing? It is also possible that multiple GPUs cannot be used for training without docker?

Due to my lack of knowledge, I take the liberty to ask you how to train with multiple GPUs.

Looking forward to your reply.

@marcocannici

Thanks for outsourcing your work. Can you kindly outsource pretrained models on classification/optical flow so that atleast we can see outputs.

Hi, thanks for your great work! The time surface is computed in a fixed neighborhood ((2R+1) x (2R+1)) in the original paper (HOTs). What is the neighborhood in your work？ Is it the whole image?