Hi,

Thank you for creating this Github! I was wondering why does forward call of GaborConstraint class has "with torch.no_grad()" (line 28 in leaf_audio_pytorch/convolution.py)? It appears the convolutional layers weight doesn't change while backpropagation because of this. However, removing it I see that weights update. Is there a reason we need torch.no_grad()? Attached a sample code to compare the Gabor layer parameters before and after training a toy code.

`

import torch
from leaf_audio_pytorch import frontend
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
a = frontend.Leaf().to(device)
## A random input that is like an audio waveform
x = torch.rand(1,1,97339).cuda(device, non_blocking=True)
a.train()
print(a._complex_conv._kernel.data)
opt = torch.optim.Adam(a.parameters(), lr=0.002, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-6, amsgrad=True)
for _ in range(100):
    b = a(x)
    ## Ground truth for the output for leaf module
    b_gt = torch.ones_like(b)
    loss = torch.norm((b-b_gt).view(1,-1),p=2,dim=-1)
    opt.zero_grad()
    loss.backward()
    opt.step()
print('====================================================')
print(a._complex_conv._kernel.data)

`

When I run the inference (like for example the one provided in the tests)

import numpy as np
import torch
import leaf_audio_pytorch.frontend as torch_frontend
py_leaf = torch_frontend.Leaf().cuda()
# (batch_size, num_samples, 1)
test_audio = np.random.random((8,15000,1)).astype(np.float32)

# convert to channel first for pytorch
t_audio = torch.Tensor(test_audio).permute(0,2,1).cuda()
print(py_leaf(t_audio))

I get this error:

RuntimeError                              Traceback (most recent call last)
<ipython-input-27-eeba248942f0> in <module>
----> 1 print(py_leaf(t_audio))

~/Envs/leaf-audio-pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

~/src/leaf-audio-pytorch/leaf_audio_pytorch/frontend.py in forward(self, x)
    131             outputs = outputs[:,:,1:]
    132 
--> 133         outputs = self._complex_conv(outputs)
    134         outputs = self._activation(outputs)
    135         outputs = self._pooling(outputs)

~/Envs/leaf-audio-pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

~/src/leaf-audio-pytorch/leaf_audio_pytorch/convolution.py in forward(self, x)
     74             kernel = torch.gather(kernel, dim=0, index=filter_order)
     75 
---> 76         filters = impulse_responses.gabor_filters(kernel, self._kernel_size, self.gabor_filter_init_t)
     77         real_filters = torch.real(filters)
     78         img_filters = torch.imag(filters)

~/src/leaf-audio-pytorch/leaf_audio_pytorch/impulse_responses.py in gabor_filters(kernel, size, t_tensor)
     33     return gabor_impulse_response(
     34         t_tensor,
---> 35         center=kernel[:, 0], fwhm=kernel[:, 1])
     36 
     37 

~/src/leaf-audio-pytorch/leaf_audio_pytorch/impulse_responses.py in gabor_impulse_response(t, center, fwhm)
      9     """Computes the gabor impulse response."""
     10     denominator = 1.0 / (np.sqrt(2.0 * math.pi) * fwhm)
---> 11     gaussian = torch.exp(torch.tensordot(1.0 / (2. * fwhm**2), -t**2, dims=0)) # TODO: validate the dims here
     12     center_frequency_complex = center.type(torch.complex64)
     13     t_complex = t.type(torch.complex64)

~/Envs/leaf-audio-pytorch/lib/python3.7/site-packages/torch/functional.py in tensordot(a, b, dims, out)
    927 
    928     if len(dims_a) == 0 or len(dims_b) == 0:
--> 929         raise RuntimeError(f"unsupported input to tensordot, got dims={dims}")
    930 
    931     if out is None:

RuntimeError: unsupported input to tensordot, got dims=0

Hi @denfed The paper says "we compute instead the convolution with 2N real-valued filters φ ̃ , n = 1, . . . , 2N , and perform squared l2-pooling". I am wandering why in the code we still use complex number when initializing the filter.

Thanks for the great work! Junjie

Thanks to @sahu-ji, this is a bug fix that allows the complex conv to be backpropagatable. I'm not sure exactly why this is a bug, because all of the operations inside of the torch.no_grad() inside the GaborConstraint are not learnable, it is only a clipping (i.e. constraint) operation. But nonetheless it does break backprop. Not anymore :)