opendatalake/detection/utils.py
def augment_detections(hyper_params, feature, label):
    """
    Augment the detection dataset.

    In your hyper_parameters.problem.augmentation add configurations to enable features.
    Supports "enable_horizontal_flip", "enable_micro_translation", "random_crop" : {"shape": { "width", "height" }}
    and "enable_texture_augmentation". Make sure to also set the "steps" otherwise this method will not be used properly.

    Random crop ensures at least one detection is in the crop region.

    Sample configuration
    "problem": {
        "augmentation": {
            "steps": 40,
            "enable_texture_augmentation": true,
            "enable_micro_translation": true,
            "enable_horizontal_flip": true,
            "random_crop": {
                "shape": {
                    "width": 256,
                    "height": 256
                }
            }
        }
    }

    :param hyper_params: The hyper parameters object
    :param feature: A dict containing all features, must be in the style created by detection datasets.
    :param label: A label dict in the detection dataset style.
    :return: Modified feature and label dict (copied & modified).
    """
    # Do not augment these ways:
    # 1) Rotation is not possible
    # 3) Scaling is not possible, because it ruins depth perception
    # However, random crops can improve performance. (Training speed and accuracy)
    if hyper_params.problem.get("augmentation", None) is None:
        return feature, label

    img_h, img_w, img_c = feature["image"].shape
    augmented_feature = {}
    augmented_label = {}
    augmented_feature["image"] = feature["image"].copy()
    if "depth" in feature:
        augmented_feature["depth"] = feature["depth"].copy()
    if "calibration" in feature:
        augmented_feature["calibration"] = feature["calibration"]
    augmented_feature["hflipped"] = np.array([0], dtype=np.uint8)
    augmented_feature["crop_offset"] = np.array([0, 0], dtype=np.int8)

    for k in label.keys():
        augmented_label[k] = [detection.copy() for detection in label[k]]

    if hyper_params.problem.augmentation.get("enable_horizontal_flip", False):
        if random.random() < 0.5:
            img_h, img_w, img_c = augmented_feature["image"].shape
            augmented_feature["image"] = np.fliplr(augmented_feature["image"])
            if "depth" in feature:
                augmented_feature["depth"] = np.fliplr(augmented_feature["depth"])
            augmented_feature["hflipped"][0] = 1
            hflip_detections(augmented_label, img_w)

    if hyper_params.problem.augmentation.get("enable_micro_translation", False):
        img_h, img_w, img_c = augmented_feature["image"].shape
        dx = int(random.random() * 3)
        dy = int(random.random() * 3)

        augmented_feature["image"] = crop_image(augmented_feature["image"], dy, dx, img_h - dy, img_w - dx)
        if "depth" in feature:
            augmented_feature["depth"] = crop_image(augmented_feature["depth"], dy, dx, img_h - dy, img_w - dx)
        augmented_feature["crop_offset"][0] += dy
        augmented_feature["crop_offset"][1] += dx

        move_detections(augmented_label, -dy, -dx)

    if hyper_params.problem.augmentation.get("random_crop", None) is not None:
        img_h, img_w, img_c = augmented_feature["image"].shape
        target_w = hyper_params.problem.augmentation.random_crop.shape.width
        target_h = hyper_params.problem.augmentation.random_crop.shape.height

        delta_x = max(int(math.ceil((target_w + 1 - img_w) / 2)), 0)
        delta_y = max(int(math.ceil((target_h + 1 - img_h) / 2)), 0)
        move_detections(augmented_label, delta_y, delta_x)
        augmented_feature["image"] = cv2.copyMakeBorder(augmented_feature["image"],
                                                        delta_y, delta_y, delta_x, delta_x,
                                                        cv2.BORDER_CONSTANT)
        img_h, img_w, img_c = augmented_feature["image"].shape

        start_x = 0
        start_y = 0
        if len(augmented_label["detections_2d"]) != 0:
            idx = random.randint(0, len(augmented_label["detections_2d"]) - 1)
            detection = augmented_label["detections_2d"][idx]
            start_x = int(detection.cx - random.random() * (target_w - 20) / 2.0 - 10)
            start_y = int(detection.cy - random.random() * (target_h - 20) / 2.0 - 10)
        else:
            start_x = int(img_w * random.random())
            start_y = int(img_h * random.random())

        # Compute start point so that crop fit's into image and random crop contains detection
        if start_x < 0:
            start_x = 0
        if start_y < 0:
            start_y = 0
        if start_x >= img_w - target_w:
            start_x = img_w - target_w - 1
        if start_y >= img_h - target_h:
            start_y = img_h - target_h - 1

        # Crop image
        augmented_feature["image"] = crop_image(augmented_feature["image"], start_y, start_x, target_h, target_w)
        if "depth" in feature:
            augmented_feature["depth"] = crop_image(augmented_feature["depth"], start_y, start_x, target_h, target_w)
        augmented_feature["crop_offset"][0] += start_y
        augmented_feature["crop_offset"][1] += start_x

        # Crop labels
        move_detections(augmented_label, -start_y, -start_x)

    if hyper_params.problem.augmentation.get("enable_texture_augmentation", False):
        if random.random() < 0.5:
            augmented_feature["image"] = full_texture_augmentation(augmented_feature["image"])

    return augmented_feature, augmented_label
def augment_detections(hyper_params, feature, label):
    """
    Augment the detection dataset.

    In your hyper_parameters.problem.augmentation add configurations to enable features.
    Supports "enable_horizontal_flip", "enable_micro_translation", "random_crop" : {"shape": { "width", "height" }}
    and "enable_texture_augmentation". Make sure to also set the "steps" otherwise this method will not be used properly.

    Random crop ensures at least one detection is in the crop region.

    Sample configuration
    "problem": {
        "augmentation": {
            "steps": 40,
            "enable_texture_augmentation": true,
            "enable_micro_translation": true,
            "enable_horizontal_flip": true,
            "random_crop": {
                "shape": {
                    "width": 256,
                    "height": 256
                }
            }
        }
    }

    :param hyper_params: The hyper parameters object
    :param feature: A dict containing all features, must be in the style created by detection datasets.
    :param label: A label dict in the detection dataset style.
    :return: Modified feature and label dict (copied & modified).
    """
    # Do not augment these ways:
    # 1) Rotation is not possible
    # 3) Scaling is not possible, because it ruins depth perception
    # However, random crops can improve performance. (Training speed and accuracy)
    if hyper_params.problem.get("augmentation", None) is None:
        return feature, label

    img_h, img_w, img_c = feature["image"].shape
    augmented_feature = {}
    augmented_label = {}
    augmented_feature["image"] = feature["image"].copy()
    if "depth" in feature:
        augmented_feature["depth"] = feature["depth"].copy()
    if "calibration" in feature:
        augmented_feature["calibration"] = feature["calibration"]
    augmented_feature["hflipped"] = np.array([0], dtype=np.uint8)
    augmented_feature["crop_offset"] = np.array([0, 0], dtype=np.int8)

    for k in label.keys():
        augmented_label[k] = [detection.copy() for detection in label[k]]

    if hyper_params.problem.augmentation.get("enable_horizontal_flip", False):
        if random.random() < 0.5:
            img_h, img_w, img_c = augmented_feature["image"].shape
            augmented_feature["image"] = np.fliplr(augmented_feature["image"])
            if "depth" in feature:
                augmented_feature["depth"] = np.fliplr(augmented_feature["depth"])
            augmented_feature["hflipped"][0] = 1
            hflip_detections(augmented_label, img_w)

    if hyper_params.problem.augmentation.get("enable_micro_translation", False):
        img_h, img_w, img_c = augmented_feature["image"].shape
        dx = int(random.random() * 3)
        dy = int(random.random() * 3)

        augmented_feature["image"] = crop_image(augmented_feature["image"], dy, dx, img_h - dy, img_w - dx)
        if "depth" in feature:
            augmented_feature["depth"] = crop_image(augmented_feature["depth"], dy, dx, img_h - dy, img_w - dx)
        augmented_feature["crop_offset"][0] += dy
        augmented_feature["crop_offset"][1] += dx

        move_detections(augmented_label, -dy, -dx)

    if hyper_params.problem.augmentation.get("random_crop", None) is not None:
        img_h, img_w, img_c = augmented_feature["image"].shape
        target_w = hyper_params.problem.augmentation.random_crop.shape.width
        target_h = hyper_params.problem.augmentation.random_crop.shape.height

        delta_x = max(int(math.ceil((target_w + 1 - img_w) / 2)), 0)
        delta_y = max(int(math.ceil((target_h + 1 - img_h) / 2)), 0)
        move_detections(augmented_label, delta_y, delta_x)
        augmented_feature["image"] = cv2.copyMakeBorder(augmented_feature["image"],
                                                        delta_y, delta_y, delta_x, delta_x,
                                                        cv2.BORDER_CONSTANT)
        img_h, img_w, img_c = augmented_feature["image"].shape

        start_x = 0
        start_y = 0
        if len(augmented_label["detections_2d"]) != 0:
            idx = random.randint(0, len(augmented_label["detections_2d"]) - 1)
            detection = augmented_label["detections_2d"][idx]
            start_x = int(detection.cx - random.random() * (target_w - 20) / 2.0 - 10)
            start_y = int(detection.cy - random.random() * (target_h - 20) / 2.0 - 10)
        else:
            start_x = int(img_w * random.random())
            start_y = int(img_h * random.random())

        # Compute start point so that crop fit's into image and random crop contains detection
        if start_x < 0:
            start_x = 0
        if start_y < 0:
            start_y = 0
        if start_x >= img_w - target_w:
            start_x = img_w - target_w - 1
        if start_y >= img_h - target_h:
            start_y = img_h - target_h - 1

        # Crop image
        augmented_feature["image"] = crop_image(augmented_feature["image"], start_y, start_x, target_h, target_w)
        if "depth" in feature:
            augmented_feature["depth"] = crop_image(augmented_feature["depth"], start_y, start_x, target_h, target_w)
        augmented_feature["crop_offset"][0] += start_y
        augmented_feature["crop_offset"][1] += start_x

        # Crop labels
        move_detections(augmented_label, -start_y, -start_x)

    if hyper_params.problem.augmentation.get("enable_texture_augmentation", False):
        if random.random() < 0.5:
            augmented_feature["image"] = full_texture_augmentation(augmented_feature["image"])

    return augmented_feature, augmented_label
Augment the detection dataset.

    In your hyper_parameters.problem.augmentation add configurations to enable features.
    Supports "enable_horizontal_flip", "enable_micro_translation", "random_crop" : {"shape": { "width", "height" }}
    and "enable_texture_augmentation". Make sure to also set the "steps" otherwise this method will not be used properly.

    Random crop ensures at least one detection is in the crop region.

    Sample configuration
    "problem": {
        "augmentation": {
            "steps": 40,
            "enable_texture_augmentation": true,
            "enable_micro_translation": true,
            "enable_horizontal_flip": true,
            "random_crop": {
                "shape": {
                    "width": 256,
                    "height": 256
                }
            }
        }
    }

    :param hyper_params: The hyper parameters object
    :param feature: A dict containing all features, must be in the style created by detection datasets.
    :param label: A label dict in the detection dataset style.
    :return: Modified feature and label dict (copied & modified).