Add the kitti evaluation code

README.md

@@ -104,6 +104,45 @@ python train.py --multiprocessing-distributed --world-size 1 --rank 0 --batch_si
     ```
     python train.py --dist-url 'tcp://IP_OF_NODE2:FREEPORT' --multiprocessing-distributed --world-size 2 --rank 1 --batch_size 64 --num_workers 8
     ```
+#### 2.3.5 Evaluation
+- **Do Evaluation on kitti val dataset**
+```
+python  train.py --evaluate  --gpu_idx 0 --pretrained_path=../checkpoints/fpn_resnet_18/fpn_resnet_18_epoch_300.pth
+```
+- **Evaluation result**
+```
+Car AP(Average Precision)@0.70, 0.70, 0.70:
+bbox AP:96.57, 89.17, 89.41
+bev  AP:97.52, 89.62, 89.78
+3d   AP:88.09, 87.86, 88.09
+aos  AP:60.28, 55.63, 55.04
+Car AP(Average Precision)@0.70, 0.50, 0.50:
+bbox AP:96.57, 89.17, 89.41
+bev  AP:98.03, 89.94, 90.09
+3d   AP:98.01, 89.94, 90.09
+aos  AP:60.28, 55.63, 55.04
+Pedestrian AP(Average Precision)@0.50, 0.50, 0.50:
+bbox AP:65.38, 64.95, 63.96
+bev  AP:68.14, 69.36, 65.41
+3d   AP:66.44, 62.36, 62.77
+aos  AP:32.13, 30.95, 30.21
+Pedestrian AP(Average Precision)@0.50, 0.25, 0.25:
+bbox AP:65.38, 64.95, 63.96
+bev  AP:88.43, 88.61, 88.59
+3d   AP:88.29, 88.45, 88.48
+aos  AP:32.13, 30.95, 30.21
+Cyclist AP(Average Precision)@0.50, 0.50, 0.50:
+bbox AP:89.62, 87.64, 87.70
+bev  AP:82.11, 75.41, 75.64
+3d   AP:80.09, 74.31, 74.45
+aos  AP:54.37, 52.01, 51.44
+Cyclist AP(Average Precision)@0.50, 0.25, 0.25:
+bbox AP:89.62, 87.64, 87.70
+bev  AP:96.04, 88.67, 88.78
+3d   AP:96.04, 88.67, 88.78
+aos  AP:54.37, 52.01, 51.44
+```
+The original evaluation code is from [here](https://github.com/traveller59/kitti-object-eval-python).
 
 #### Tensorboard
 

requirements.txt

@@ -9,3 +9,4 @@ scikit-learn==0.22.2
 wget==3.2
 tqdm==4.54.0
 matplotlib==3.3.3
+numba

sfa/train.py

@@ -40,6 +40,10 @@
 from config.train_config import parse_train_configs
 from losses.losses import Compute_Loss
 
+from utils.torch_utils import _sigmoid
+from utils.evaluation_utils import decode, post_processing, draw_predictions, convert_det_to_real_values, convert_detection_to_kitti_annos
+from utils import kitti_common
+from utils.eval import get_official_eval_result
 
 def main():
     configs = parse_train_configs()
@@ -250,8 +254,9 @@ def train_one_epoch(train_dataloader, model, optimizer, lr_scheduler, epoch, con
 def validate(val_dataloader, model, configs):
     losses = AverageMeter('Loss', ':.4e')
     criterion = Compute_Loss(device=configs.device)
-    # switch to train mode
+
     model.eval()
+    detections_list = []
     with torch.no_grad():
         for batch_idx, batch_data in enumerate(tqdm(val_dataloader)):
             metadatas, imgs, targets = batch_data
@@ -270,6 +275,23 @@ def validate(val_dataloader, model, configs):
             else:
                 reduced_loss = total_loss.data
             losses.update(to_python_float(reduced_loss), batch_size)
+            outputs['hm_cen'] = _sigmoid(outputs['hm_cen'])
+            outputs['cen_offset'] = _sigmoid(outputs['cen_offset'])            
+
+
+            for idx in range(outputs['hm_cen'].shape[0]):
+                # detections size (batch_size, K, 10)
+                detections = decode(outputs['hm_cen'][idx:idx+1,:], outputs['cen_offset'][idx:idx+1,:],
+                                    outputs['direction'][idx:idx+1,:], outputs['z_coor'][idx:idx+1,:],
+                                    outputs['dim'][idx:idx+1,:], K=configs.K)
+                detections = detections.cpu().numpy().astype(np.float32)
+                detections = post_processing(detections, configs.num_classes, configs.down_ratio)               
+                detections_list.append(detections[0])
+
+        dt_annos = convert_detection_to_kitti_annos(detections_list, val_dataloader.dataset)
+        gt_annos = kitti_common.get_label_annos(val_dataloader.dataset.label_dir, val_dataloader.dataset.sample_id_list)
+        print("Doing evaluation")
+        print(get_official_eval_result(gt_annos, dt_annos, [i for i in range(configs.num_classes)])) 
 
     return losses.avg
 

sfa/utils/box_np_ops.py

@@ -0,0 +1,94 @@
+import numba
+import numpy as np
+
+
+def corners_nd(dims, origin=0.5):
+    """generate relative box corners based on length per dim and
+    origin point. 
+
+    Args:
+        dims (float array, shape=[N, ndim]): array of length per dim
+        origin (list or array or float): origin point relate to smallest point.
+
+    Returns:
+        float array, shape=[N, 2 ** ndim, ndim]: returned corners. 
+        point layout example: (2d) x0y0, x0y1, x1y0, x1y1;
+            (3d) x0y0z0, x0y0z1, x0y1z0, x0y1z1, x1y0z0, x1y0z1, x1y1z0, x1y1z1
+            where x0 < x1, y0 < y1, z0 < z1
+    """
+    ndim = int(dims.shape[1])
+    corners_norm = np.stack(
+        np.unravel_index(np.arange(2**ndim), [2] * ndim),
+        axis=1).astype(dims.dtype)
+    # now corners_norm has format: (2d) x0y0, x0y1, x1y0, x1y1
+    # (3d) x0y0z0, x0y0z1, x0y1z0, x0y1z1, x1y0z0, x1y0z1, x1y1z0, x1y1z1
+    # so need to convert to a format which is convenient to do other computing.
+    # for 2d boxes, format is clockwise start with minimum point
+    # for 3d boxes, please draw lines by your hand.
+    if ndim == 2:
+        # generate clockwise box corners
+        corners_norm = corners_norm[[0, 1, 3, 2]]
+    elif ndim == 3:
+        corners_norm = corners_norm[[0, 1, 3, 2, 4, 5, 7, 6]]
+    corners_norm = corners_norm - np.array(origin, dtype=dims.dtype)
+    corners = dims.reshape([-1, 1, ndim]) * corners_norm.reshape(
+        [1, 2**ndim, ndim])
+    return corners
+
+
+def rotation_3d_in_axis(points, angles, axis=0):
+    # points: [N, point_size, 3]
+    rot_sin = np.sin(angles)
+    rot_cos = np.cos(angles)
+    ones = np.ones_like(rot_cos)
+    zeros = np.zeros_like(rot_cos)
+    if axis == 1:
+        rot_mat_T = np.stack([[rot_cos, zeros, -rot_sin], [zeros, ones, zeros],
+                              [rot_sin, zeros, rot_cos]])
+    elif axis == 2 or axis == -1:
+        rot_mat_T = np.stack([[rot_cos, -rot_sin, zeros],
+                              [rot_sin, rot_cos, zeros], [zeros, zeros, ones]])
+    elif axis == 0:
+        rot_mat_T = np.stack([[zeros, rot_cos, -rot_sin],
+                              [zeros, rot_sin, rot_cos], [ones, zeros, zeros]])
+    else:
+        raise ValueError("axis should in range")
+
+    return np.einsum('aij,jka->aik', points, rot_mat_T)
+
+
+def center_to_corner_box3d(centers,
+                           dims,
+                           angles=None,
+                           origin=(0.5, 0.5, 0.5),
+                           axis=2):
+    """convert kitti locations, dimensions and angles to corners
+
+    Args:
+        centers (float array, shape=[N, 3]): locations in kitti label file.
+        dims (float array, shape=[N, 3]): dimensions in kitti label file.
+        angles (float array, shape=[N]): rotation_y in kitti label file.
+        origin (list or array or float): origin point relate to smallest point.
+            use [0.5, 1.0, 0.5] in camera and [0.5, 0.5, 0] in lidar.
+        axis (int): rotation axis. 1 for camera and 2 for lidar.
+    Returns:
+        [type]: [description]
+    """
+    # 'length' in kitti format is in x axis.
+    # yzx(hwl)(kitti label file)<->xyz(lhw)(camera)<->z(-x)(-y)(wlh)(lidar)
+    # center in kitti format is [0.5, 1.0, 0.5] in xyz.
+    corners = corners_nd(dims, origin=origin)
+    # corners: [N, 8, 3]
+    if angles is not None:
+        corners = rotation_3d_in_axis(corners, angles, axis=axis)
+    corners += centers.reshape([-1, 1, 3])
+    return corners
+
+
+def project_to_image(points_3d, proj_mat):
+    points_shape = list(points_3d.shape)
+    points_shape[-1] = 1
+    points_4 = np.concatenate([points_3d, np.zeros(points_shape)], axis=-1)
+    point_2d = points_4 @ proj_mat.T
+    point_2d_res = point_2d[..., :2] / point_2d[..., 2:3]
+    return point_2d_res