%load_ext autoreload
%autoreload 2

os.environ["CUDA_VISIBLE_DEVICES"] = '1'

cfg = OmegaConf.load(f'../config/experiment/msp300_1.yaml')

model, post_proc, psf, micro, img_3d, decode_dl = load_all(cfg, False)

/groups/turaga/home/speisera/anaconda3/envs/decode2_dev/lib/python3.7/site-packages/torch/cuda/__init__.py:81: UserWarning: 
    Found GPU2 NVS 510 which is of cuda capability 3.0.
    PyTorch no longer supports this GPU because it is too old.
    The minimum cuda capability that we support is 3.5.
    
  warnings.warn(old_gpu_warn % (d, name, major, capability[1]))

Crop size larger than volume in at least one dimension. Crop size changed to (37, 48, 48)
1 volumes

image_paths = sorted(glob.glob(str(Path(cfg.data_path.image_path).parent)+'/msp300*.tif'))
image_paths

['/groups/turaga/home/speisera/Mackebox/Artur/WorkDB/deepstorm/datasets/CodFish/smFISH_data_Titlow/msp300_smFISH_1.tif',
 '/groups/turaga/home/speisera/Mackebox/Artur/WorkDB/deepstorm/datasets/CodFish/smFISH_data_Titlow/msp300_smFISH_2.tif',
 '/groups/turaga/home/speisera/Mackebox/Artur/WorkDB/deepstorm/datasets/CodFish/smFISH_data_Titlow/msp300_smFISH_3.tif',
 '/groups/turaga/home/speisera/Mackebox/Artur/WorkDB/deepstorm/datasets/CodFish/smFISH_data_Titlow/msp300_smFISH_4.tif']

predict[source]

predict(model, post_proc, image_paths, window_size=[128, 256, 256], device='cuda')

pred_df = predict(model, image_paths)

  0%|          | 0/4 [00:00<?, ?it/s]

msp300_smFISH_1.tif

  0%|          | 0/4 [00:02<?, ?it/s]

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-9-cc1db0739a8b> in <module>
----> 1 pred_df = predict(model, image_paths)

<ipython-input-8-c6b4e5cb344e> in predict(model, image_paths, window_size, device)
     10             img = img.reshape(-1,z,y,x)
     11             for i in range(len(img)):
---> 12                 output = sliding_window_inference(img[i][None,None], window_size, 1, model.cuda(), overlap=0.2, sw_device=device, device='cpu', mode='gaussian')
     13                 output = model.tensor_to_dict(output)
     14                 p_si = sliding_window_inference(output['logits'], window_size, 1, post_proc, overlap=0.2, sw_device=device, device='cpu', mode='gaussian')

~/anaconda3/envs/decode2_dev/lib/python3.7/site-packages/monai/inferers/utils.py in sliding_window_inference(inputs, roi_size, sw_batch_size, predictor, overlap, mode, sigma_scale, padding_mode, cval, sw_device, device, *args, **kwargs)
    128         ]
    129         window_data = torch.cat([inputs[win_slice] for win_slice in unravel_slice]).to(sw_device)
--> 130         seg_prob = predictor(window_data, *args, **kwargs).to(device)  # batched patch segmentation
    131 
    132         if not _initialized:  # init. buffer at the first iteration

~/anaconda3/envs/decode2_dev/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/Dropbox (mackelab)/Artur/WorkDB/deepstorm/decode_fish/decode_fish/engine/model.py in forward(self, x)
    511 
    512     def forward(self, x):
--> 513         out =  self.unet(x)
    514 
    515         logit    = F.elu(self.p_out1(out))

~/anaconda3/envs/decode2_dev/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/Dropbox (mackelab)/Artur/WorkDB/deepstorm/decode_fish/decode_fish/engine/model.py in forward(self, x)
    444             # pass the output from the corresponding encoder and the output
    445             # of the previous decoder
--> 446             x = decoder(encoder_features, x)
    447 
    448 #         x = self.final_conv(x)

~/anaconda3/envs/decode2_dev/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/Dropbox (mackelab)/Artur/WorkDB/deepstorm/decode_fish/decode_fish/engine/model.py in forward(self, encoder_features, x)
    246     def forward(self, encoder_features, x):
    247         x = self.upsampling(encoder_features=encoder_features, x=x)
--> 248         x = self.joining(encoder_features, x)
    249         x = self.basic_module(x)
    250         return x

~/Dropbox (mackelab)/Artur/WorkDB/deepstorm/decode_fish/decode_fish/engine/model.py in _joining(encoder_features, x, concat)
    253     def _joining(encoder_features, x, concat):
    254         if concat:
--> 255             return torch.cat((encoder_features, x), dim=1)
    256         else:
    257             return encoder_features + x

RuntimeError: CUDA out of memory. Tried to allocate 3.00 GiB (GPU 0; 11.78 GiB total capacity; 3.28 GiB already allocated; 2.07 GiB free; 3.53 GiB reserved in total by PyTorch)

with torch.no_grad():
    output = sliding_window_inference(img[None], [37,128,128], 1, model.cuda(), overlap=0.2, sw_device ='cuda', device='cpu', mode='gaussian')
    output = model.tensor_to_dict(output)
    p_si = sliding_window_inference(output['logits'], [37,128,128], 1, post_proc, overlap=0.2, sw_device ='cuda', device='cpu', mode='gaussian')
    pred_df = post_proc.get_df(output, p_si)

with torch.no_grad():
    model.cpu()
    res = model.tensor_to_dict(model(img[None]))
    gt_df = post_proc.get_df(res)
    free_mem()

gt_df[:10]

pred_df[:10]

print(len(gt_df))
print(len(pred_df))

!nbdev_build_lib

Converted 00_models.ipynb.
Converted 01_psf.ipynb.
Converted 02_microscope.ipynb.
Converted 03_noise.ipynb.
Converted 04_pointsource.ipynb.
Converted 05_gmm_loss.ipynb.
Converted 06_plotting.ipynb.
Converted 07_file_io.ipynb.
Converted 08_dataset.ipynb.
Converted 09_output_trafo.ipynb.
Converted 10_evaluation.ipynb.
Converted 11_emitter_io.ipynb.
Converted 12_utils.ipynb.
Converted 13_train.ipynb.
Converted 15_fit_psf.ipynb.
Converted 16_visualization.ipynb.
Converted 17_eval_routines.ipynb.
Converted 18_predict_funcs.ipynb.
Converted index.ipynb.