Will you release the code of webcam demo?Thanks about clean-pvnet

[MinchangSung0223]

I created a demo using a realsense sr305 camera.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from numpy.ctypeslib import ndpointer
import argparse
import json
import os
import time
import numpy as np
import math
import threading
import time
from visualization import Visualizer2D as vis

from lib.config import cfg, args
from lib.networks import make_network
from lib.datasets import make_data_loader
from lib.utils.net_utils import load_network
from numpy.ctypeslib import ndpointer
best_center=[0 ,0]
center=[0 ,0]
prev_q_value=0
q_value=0
prev_depth=0
#-----------------------------------------------------------------------------

#import matplotlib.pyplot as plt
import ctypes
import numpy as np
import cv2
import os
import threading
import tqdm
import torch
from lib.visualizers import make_visualizer
import pycocotools.coco as coco
from lib.datasets import make_data_loader
from lib.evaluators import make_evaluator
import tqdm
import torch
from lib.utils.pvnet import pvnet_pose_utils
from lib.networks import make_network
import matplotlib.patches as patches
from lib.utils.net_utils import load_network
from ctypes import cdll
import pyrealsense2 as rs
#---------------------------------------realsense---------------------------------------------
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

camera_intr_filename = "realsense.intr"
camera_intr = CameraIntrinsics.load(camera_intr_filename)
def detect_img():
   global color_img
   global mask
   global corner_2d_pred
   global depth_im
   global color_im
   global segmask
   global center
   global angle
   global best_angle
   global prev_q_value
   global best_center
   def nothing(x):
      pass
   mask = np.zeros((640,480),dtype=np.uint8)
   cv2.namedWindow("rgb", cv2.WINDOW_NORMAL)
   cv2.resizeWindow("rgb", 1280,720)
   cv2.createTrackbar('W', "rgb", 0, 100, nothing)
   cv2.namedWindow("mask", cv2.WINDOW_NORMAL)
   cv2.resizeWindow("mask", 640,480)
   cv2.namedWindow("depth", cv2.WINDOW_NORMAL)
   cv2.resizeWindow("depth", 1000,1000)
   line_arr1 = [0, 1, 3, 2, 0, 4, 6, 2]
   line_arr2 = [5, 4, 6, 7, 5, 1, 3, 7]

   divide_value = 65535
   pipeline.start(config)
   while 1:
    try:
     frames = pipeline.wait_for_frames()
     w = cv2.getTrackbarPos('W',"rgb")
     depth_frame = frames.get_depth_frame()
     color_frame = frames.get_color_frame()

     if not depth_frame or not color_frame:
           continue
     depth_to_color_img = np.asanyarray(depth_frame.get_data())/divide_value
     color_img = np.asanyarray(color_frame.get_data())
     print(depth_to_color_img)
     depth_im =DepthImage(depth_to_color_img.astype("float32"), frame=camera_intr.frame)
     color_im = ColorImage(np.zeros([480, 640,
                                   3]).astype(np.uint8),
                         frame=camera_intr.frame)
     seg_mask =cv2.resize(mask.copy(),(640,480),interpolation=cv2.INTER_CUBIC)
     segmask = BinaryImage(seg_mask)
     show_img = cv2.resize(color_img.copy(),(640,480),interpolation=cv2.INTER_CUBIC)
     for i in range(len(line_arr1)-1):
        show_img = cv2.line(show_img, (int(corner_2d_pred[line_arr1[i]][0]),int(corner_2d_pred[line_arr1[i]][1])),(int(corner_2d_pred[line_arr1[i+1]][0]),int(corner_2d_pred[line_arr1[i+1]][1])),(255,0,0),1)
     for i in range(len(line_arr2)-1):
        show_img = cv2.line(show_img, (int(corner_2d_pred[line_arr2[i]][0]),int(corner_2d_pred[line_arr2[i]][1])),(int(corner_2d_pred[line_arr2[i+1]][0]),int(corner_2d_pred[line_arr2[i+1]][1])),(255,0,0),1)
     #print(center_2d_pred[0])
     show_img = cv2.circle(show_img, (int(center_2d_pred[0][0]),int(center_2d_pred[0][1])),1,(0,255,0),3)
     show_img = cv2.line(show_img, (int(center_2d_pred[0][0]),int(center_2d_pred[0][1])),(int(center_2d_pred[1][0]),int(center_2d_pred[1][1])),(255,0,0),2)
     show_img = cv2.line(show_img, (int(center_2d_pred[0][0]),int(center_2d_pred[0][1])),(int(center_2d_pred[2][0]),int(center_2d_pred[2][1])),(0,255,0),2)
     show_img = cv2.line(show_img, (int(center_2d_pred[0][0]),int(center_2d_pred[0][1])),(int(center_2d_pred[3][0]),int(center_2d_pred[3][1])),(0,0,255),2)
     try:
        ret, img_binary = cv2.threshold(mask, 127, 255, 0)
        contours, color_hierachy =cv2.findContours(img_binary.copy(),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
        c0 = np.reshape(contours[0],(-1,2))
        show_img = cv2.drawContours(np.uint8(show_img), contours, -1, (0,255,0), 1)
     except:
        pass
     show_img_resize = cv2.resize( show_img.copy(),(640,480),interpolation=cv2.INTER_CUBIC)
     print(corner_2d_pred[0])
     try:
        r=50
        show_img_resize = cv2.line(show_img_resize, (int(best_center[0]-r*math.cos(best_angle/180*math.pi)),int(best_center[1]-r*math.sin(best_angle/180*math.pi))), (int(best_center[0]+r*math.cos(best_angle/180*math.pi)),int(best_center[1]+r*math.sin(best_angle/180*math.pi))), (0,0,255), 5)
        show_img_resize = cv2.line(show_img_resize, (int(best_center[0]),int(best_center[1])), (int(best_center[0]+1),int(best_center[1])) , (0,136,255), 7)
        show_img_resize = cv2.line(show_img_resize, (int(center[0]-r*math.cos(angle/180*math.pi)),int(center[1]-r*math.sin(angle/180*math.pi))), (int(center[0]+r*math.cos(angle/180*math.pi)),int(center[1]+r*math.sin(angle/180*math.pi))), (255,100,255), 5)
        show_img_resize = cv2.line(show_img_resize, (int(center[0]),int(center[1])), (int(center[0]+1),int(center[1])) , (100,136,255), 7)
      
     except:
      	 pass
     depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_to_color_img*1000, alpha=2), 1)
     cv2.imshow("rgb",show_img_resize)
     cv2.imshow("depth",depth_colormap)
     cv2.imshow("mask",seg_mask)
     k = cv2.waitKey(5) & 0xFF
     if k == ord('s'):
        cv2.destroyWindow("rgb")
    except Exception as inst:
     print(inst)
     pass
   end()
def run_pvnet():
   network = make_network(cfg).cuda()
   load_network(network, cfg.model_dir, epoch=cfg.test.epoch)
   network.eval()
   visualizer = make_visualizer(cfg)
   data_loader = make_data_loader(cfg, is_train=False)
   evaluator = make_evaluator(cfg)
   mean=[0.485, 0.456, 0.406]
   std=[0.229, 0.224, 0.225]
   global corner_2d_pred
   global center_2d_pred
   global mask
   global pose_pred
   fps_3d=np.load("fps_3d.npy")
   center_3d=np.load("center_3d.npy")
   corner_3d=np.load("corner_3d.npy")
   prev_pose_pred=np.zeros((4,4))
  # K = np.array([[700.0, 0.0, 320.0], [0.0, 700.0, 240.0], [0.0, 0.0, 1.0]])
   #K = np.array([[2.40691932e+03, 0.00000000e+00, 3.19065470e+02],[0.00000000e+00, 2.71622117e+03, 2.03018751e+02],[0.00000000e+00, 0.00000000e+00, 1.00000000e+00]])
   K = np.array([[608.13312805 ,  0.      ,   324.97036511],[  0.     ,    612.61549006 ,242.14119643],[  0.       ,    0.        ,   1.        ]])

   while 1:
     try:
       start = time.time()
       color_img_= (cv2.resize(color_img,(640,480),interpolation=cv2.INTER_CUBIC)/255-mean)/std
       color_img__ = np.zeros((1,3,480,640),dtype=float)
       color_img__[0,2,:,:] = color_img_[:,:,0]
       color_img__[0,1,:,:] = color_img_[:,:,1]
       color_img__[0,0,:,:] = color_img_[:,:,2]
       inp = torch.reshape(torch.from_numpy(color_img__),(1,3,480,640)).type(torch.FloatTensor).cuda()
       with torch.no_grad():
        try:
           output = network(inp)
        except:
           pass

       #print(output['mask'].shape)
       kpt_2d = output['kpt_2d'][0].detach().cpu().numpy()
       
   
       kpt_3d = np.concatenate([fps_3d, [center_3d]], axis=0)

       #K = np.array([[700.0, 0.0, 320.0], [0.0, 700.0, 240.0], [0.0, 0.0, 1.0]])
       alpha = 0.99
       now_pose_pred = pvnet_pose_utils.pnp(kpt_3d, kpt_2d, K)
       pose_pred = now_pose_pred 
      # for j in range(0,3):
      #    for k in range(0,4):
      #       pose_pred[j,k] = 0.0*prev_pose_pred[j,k]+1*now_pose_pred[j,k]
       
      # prev_pose_pred = now_pose_pred
       #print("pose_pred ::::: \n",pose_pred)
       center3d = [[np.mean(corner_3d[:,0]),np.mean(corner_3d[:,1]),np.mean(corner_3d[:,2])],
[np.mean(corner_3d[:,0])+0.05,np.mean(corner_3d[:,1]),np.mean(corner_3d[:,2])],
[np.mean(corner_3d[:,0]),np.mean(corner_3d[:,1])-0.05,np.mean(corner_3d[:,2])],
[np.mean(corner_3d[:,0]),np.mean(corner_3d[:,1]),np.mean(corner_3d[:,2])+0.05]]
       corner_2d_pred = pvnet_pose_utils.project(corner_3d, K, pose_pred)
       center_2d_pred = pvnet_pose_utils.project(center3d, K, pose_pred)

       showimg= np.uint8((color_img_*std+mean)*255)
       showimg_= np.zeros((480,640,3),dtype=np.uint8)
       showimg_[:,:,0] = showimg[:,:,2]
       showimg_[:,:,1] = showimg[:,:,1]
       showimg_[:,:,2] = showimg[:,:,0]

       mask = np.transpose(np.transpose(np.uint8(np.reshape(output['mask'][0].detach().cpu().numpy(),(480,640))))*255)
       print("TIME :",time.time()-start)
     except:
       break;

       pass
   end()
   evaluator.summarize()
if __name__ == '__main__':
   t1 = threading.Thread(target=detect_img)
   t1.start()
   t2 = threading.Thread(target=run_pvnet)
   t2.start()

   ```
-------------------------------------------------------------------------------------------------------------------------------------
You need three files.

fps_3d.npy
center_3d.npy
corner_3d.npy


python3 test_realsense.py --type visualize --cfg_file configs/linemod.yaml train.dataset CustomTrain test.dataset CustomTrain model mycat

[pengsida]

Yes. I plan to release the code of webcam demo. But our lab is closed now due to the infectious disease, so I do not know when I can release the code.

[kevin860213]

I'm sory to hear that. Hope it will be better asap. Thanks for your help.

[kevin860213]

Hi, i saw you updatesd the Readme, i found you updated the tless data. Did you update the webcam demo at the same time? Or i missed something? thanks your help

[pengsida]

The webcam demo requires the GUI environment, which I don't have currently since our lab is still closed.

[kevin860213]

Thanks your help.

[mandywoo]

@tjdalsckd How did you get the mean and std values?

[pengsida]

https://github.com/zju3dv/clean-pvnet/blob/master/lib/datasets/transforms.py#L88

[Harrydsk]

python3 test_realsense.py --type visualize --cfg_file configs/linemod.yaml train.dataset CustomTrain test.dataset CustomTrain model mycat

what are this "train.dataset CustomTrain test.dataset CustomTrain model mycat"