Hi, @s-gupta, I have a question about the MS-COCO version, since there are two versions for it, I need to download the MS-COCO 2014 or 2017?

And where to download the file 'instances_vcoco_all_2014.json''? Thanks!!

np.random.seed(1)
positive_index = np.where(vcoco['label'] == 1)[0]
positive_index = np.random.permutation(positive_index)

# the demo here laods images from the COCO website, 
# you can alternatively use your own local folder of COCO images.
load_coco_image_from_web = True
if load_coco_image_from_web:
    base_coco_url = 'http://mscoco.org/images/'
    from PIL import Image
    #import urllib, cStringIO
    import urllib.request
    from io import StringIO

cc = plt.get_cmap('hsv', lut=4)

for i in range(5):
    id = positive_index[i]

    # load image
    coco_image = coco.loadImgs(ids=[vcoco['image_id'][id][0]])[0]
    if load_coco_image_from_web:
        coco_url = base_coco_url + str(coco_image['id'])
        print(coco_url)
        #file = cStringIO.StringIO(urllib.urlopen(coco_url).read())
        file = StringIO(urllib.request.urlopen(coco_url).read())
        im = np.asarray(Image.open(file))
        
    sy = 4.; sx = float(im.shape[1])/float(im.shape[0])*sy;
    fig, ax = subplot(plt, (1,1), (sy,sx)); ax.set_axis_off(); 
    ax.imshow(im);
    
    # draw bounding box for agent
    draw_bbox(plt, ax, vcoco['bbox'][[id],:], edgecolor=cc(0)[:3])
    role_bbox = vcoco['role_bbox'][id,:]*1.
    role_bbox = role_bbox.reshape((-1,4))
    for j in range(1, len(vcoco['role_name'])):
        if not np.isnan(role_bbox[j,0]):
            draw_bbox(plt, ax, role_bbox[[j],:], edgecolor=cc(j)[:3])

I get this error:

http://mscoco.org/images/554541
---------------------------------------------------------------------------
HTTPError                                 Traceback (most recent call last)
<ipython-input-19-7ebab5b666ed> in <module>
     24         print(coco_url)
     25         #file = cStringIO.StringIO(urllib.urlopen(coco_url).read())
---> 26         file = StringIO(urllib.request.urlopen(coco_url).read())
     27         im = np.asarray(Image.open(file))
     28 

/usr/lib/python3.8/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
--> 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

/usr/lib/python3.8/urllib/request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
--> 531             response = meth(req, response)
    532 
    533         return response

/usr/lib/python3.8/urllib/request.py in http_response(self, request, response)
    638         # request was successfully received, understood, and accepted.
    639         if not (200 <= code < 300):
--> 640             response = self.parent.error(
    641                 'http', request, response, code, msg, hdrs)
    642 

/usr/lib/python3.8/urllib/request.py in error(self, proto, *args)
    567         if http_err:
    568             args = (dict, 'default', 'http_error_default') + orig_args
--> 569             return self._call_chain(*args)
    570 
    571 # XXX probably also want an abstract factory that knows when it makes

/usr/lib/python3.8/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
--> 502             result = func(*args)
    503             if result is not None:
    504                 return result

/usr/lib/python3.8/urllib/request.py in http_error_default(self, req, fp, code, msg, hdrs)
    647 class HTTPDefaultErrorHandler(BaseHandler):
    648     def http_error_default(self, req, fp, code, msg, hdrs):
--> 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    650 
    651 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 503: Service Unavailable

the base URL is broken. could you please suggest fixes?

(vcoco) mona@goku:~/research/code/v-coco$ python demo_vcoco.py 
added /home/mona/research/code/v-coco/coco/PythonAPI to pythonpath
loading annotations into memory...
Done (t=1.12s)
creating index...
index created!
Traceback (most recent call last):
  File "demo_vcoco.py", line 28, in <module>
    vcoco_all = vu.load_vcoco('vcoco_train')
  File "/home/mona/research/code/v-coco/vsrl_utils.py", line 46, in load_vcoco
    np.array(vsrl_data[i]['role_object_id']).reshape((len(vsrl_data[i]['role_name']),-1)).T
KeyError: 'role_object_id'

here is the code:

import __init__
import vsrl_utils as vu
import numpy as np

import matplotlib
import matplotlib.pyplot as plt

def draw_bbox(plt, ax, rois, fill=False, linewidth=2, edgecolor=[1.0, 0.0, 0.0], **kwargs):
    for i in range(rois.shape[0]):
        roi = rois[i,:].astype(np.int)
        ax.add_patch(plt.Rectangle((roi[0], roi[1]),
            roi[2] - roi[0], roi[3] - roi[1],
            fill=False, linewidth=linewidth, edgecolor=edgecolor, **kwargs))

##def subplot(plt, (Y, X), (sz_y, sz_x) = (10, 10)):
def subplot(plt, yx, sz = (10, 10)):
    (Y, X) = yx
    (sz_y, sz_x) = sz
    plt.rcParams['figure.figsize'] = (X*sz_x, Y*sz_y)
    fig, axes = plt.subplots(Y, X)
    return fig, axes


# Load COCO annotations for V-COCO images
coco = vu.load_coco()

# Load the VCOCO annotations for vcoco_train image set
vcoco_all = vu.load_vcoco('vcoco_train')
for x in vcoco_all:
    x = vu.attach_gt_boxes(x, coco)

Hello

As the dataset does not provide automatically the object class annotation but object class ID, I try to get the object class name from mscoco annotations, however object class IDs does not align with mscoco annotations. I get weird tuples like <snowboard, cup>. Is there a way to get it?

Thank you @s-gupta to cretae this project, I have several questions about json data on folder data/vcoco

1. Are you create object class and verbs class on that data by yourself using data from COCO?
2. I want to train your data annotations using this project , so I must adjust your annotations to fit that project. Can you explain what meaning of the data on category; 'label', 'ann_id' and 'role_object_id' on every your json file?

Thank you for your help :)