How to straighten a rotated rectangle area of an image using OpenCV in Python?

This is my C++ version that performs the same task. I have noticed it is a bit slow. If anyone sees anything that would improve the performance of this function, then please let me know. 🙂

bool extractPatchFromOpenCVImage( cv::Mat& src, cv::Mat& dest, int x, int y, double angle, int width, int height) {

  // obtain the bounding box of the desired patch
  cv::RotatedRect patchROI(cv::Point2f(x,y), cv::Size2i(width,height), angle);
  cv::Rect boundingRect = patchROI.boundingRect();

  // check if the bounding box fits inside the image
  if ( boundingRect.x >= 0 && boundingRect.y >= 0 &&
       (boundingRect.x+boundingRect.width) < src.cols &&  
       (boundingRect.y+boundingRect.height) < src.rows ) { 

    // crop out the bounding rectangle from the source image
    cv::Mat preCropImg = src(boundingRect);

    // the rotational center relative tot he pre-cropped image
    int cropMidX, cropMidY;
    cropMidX = boundingRect.width/2;
    cropMidY = boundingRect.height/2;

    // obtain the affine transform that maps the patch ROI in the image to the
    // dest patch image. The dest image will be an upright version.
    cv::Mat map_mat = cv::getRotationMatrix2D(cv::Point2f(cropMidX, cropMidY), angle, 1.0f);
    map_mat.at<double>(0,2) += static_cast<double>(width/2 - cropMidX);
    map_mat.at<double>(1,2) += static_cast<double>(height/2 - cropMidY);

    // rotate the pre-cropped image. The destination image will be
    // allocated by warpAffine()
    cv::warpAffine(preCropImg, dest, map_mat, cv::Size2i(width,height)); 

    return true;
  } // if
  else {
    return false;
  } // else
} // extractPatch

I had problems with wrong offsets while using the solutions here and in similar questions.

So I did the math and came up with the following solution that works:

def subimage(self,image, center, theta, width, height):
    theta *= 3.14159 / 180 # convert to rad

    v_x = (cos(theta), sin(theta))
    v_y = (-sin(theta), cos(theta))
    s_x = center[0] - v_x[0] * ((width-1) / 2) - v_y[0] * ((height-1) / 2)
    s_y = center[1] - v_x[1] * ((width-1) / 2) - v_y[1] * ((height-1) / 2)

    mapping = np.array([[v_x[0],v_y[0], s_x],
                        [v_x[1],v_y[1], s_y]])

    return cv2.warpAffine(image,mapping,(width, height),flags=cv2.WARP_INVERSE_MAP,borderMode=cv2.BORDER_REPLICATE)

For reference here is an image that explains the math behind it:

Note that

w_dst = width-1
h_dst = height-1

This is because the last coordinate has the value width-1 and not width, or height.

Similar recipe for openCV version 3.4.0.

from cv2 import cv
import numpy as np

def getSubImage(rect, src):
    # Get center, size, and angle from rect
    center, size, theta = rect
    # Convert to int 
    center, size = tuple(map(int, center)), tuple(map(int, size))
    # Get rotation matrix for rectangle
    M = cv2.getRotationMatrix2D( center, theta, 1)
    # Perform rotation on src image
    dst = cv2.warpAffine(src, M, src.shape[:2])
    out = cv2.getRectSubPix(dst, size, center)
    return out

img = cv2.imread('img.jpg')
# Find some contours
thresh2, contours, hierarchy = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# Get rotated bounding box
rect = cv2.minAreaRect(contours[0])
# Extract subregion
out = getSubImage(rect, img)
# Save image
cv2.imwrite('out.jpg', out)

The other methods will work only if the content of the rectangle is in the rotated image after rotation and will fail badly in other situations. What if some of the part are lost? See an example below:

If you are to crop the rotated rectangle text area using the above method,

import cv2
import numpy as np


def main():
    img = cv2.imread("big_vertical_text.jpg")
    cnt = np.array([
            [[64, 49]],
            [[122, 11]],
            [[391, 326]],
            [[308, 373]]
        ])
    print("shape of cnt: {}".format(cnt.shape))
    rect = cv2.minAreaRect(cnt)
    print("rect: {}".format(rect))

    box = cv2.boxPoints(rect)
    box = np.int0(box)

    print("bounding box: {}".format(box))
    cv2.drawContours(img, [box], 0, (0, 0, 255), 2)

    img_crop, img_rot = crop_rect(img, rect)

    print("size of original img: {}".format(img.shape))
    print("size of rotated img: {}".format(img_rot.shape))
    print("size of cropped img: {}".format(img_crop.shape))

    new_size = (int(img_rot.shape[1]/2), int(img_rot.shape[0]/2))
    img_rot_resized = cv2.resize(img_rot, new_size)
    new_size = (int(img.shape[1]/2)), int(img.shape[0]/2)
    img_resized = cv2.resize(img, new_size)

    cv2.imshow("original contour", img_resized)
    cv2.imshow("rotated image", img_rot_resized)
    cv2.imshow("cropped_box", img_crop)

    # cv2.imwrite("crop_img1.jpg", img_crop)
    cv2.waitKey(0)


def crop_rect(img, rect):
    # get the parameter of the small rectangle
    center = rect[0]
    size = rect[1]
    angle = rect[2]
    center, size = tuple(map(int, center)), tuple(map(int, size))

    # get row and col num in img
    height, width = img.shape[0], img.shape[1]
    print("width: {}, height: {}".format(width, height))

    M = cv2.getRotationMatrix2D(center, angle, 1)
    img_rot = cv2.warpAffine(img, M, (width, height))

    img_crop = cv2.getRectSubPix(img_rot, size, center)

    return img_crop, img_rot


if __name__ == "__main__":
    main()

This is what you will get:

Apparently, some of the parts are cut out! Why do not directly warp the rotated rectangle since we can get its four corner points with cv.boxPoints() method?

import cv2
import numpy as np


def main():
    img = cv2.imread("big_vertical_text.jpg")
    cnt = np.array([
            [[64, 49]],
            [[122, 11]],
            [[391, 326]],
            [[308, 373]]
        ])
    print("shape of cnt: {}".format(cnt.shape))
    rect = cv2.minAreaRect(cnt)
    print("rect: {}".format(rect))

    box = cv2.boxPoints(rect)
    box = np.int0(box)
    width = int(rect[1][0])
    height = int(rect[1][1])

    src_pts = box.astype("float32")
    dst_pts = np.array([[0, height-1],
                        [0, 0],
                        [width-1, 0],
                        [width-1, height-1]], dtype="float32")
    M = cv2.getPerspectiveTransform(src_pts, dst_pts)
    warped = cv2.warpPerspective(img, M, (width, height))

Now the cropped image becomes

Much better, isn’t it? If you check carefully, you will notice that there are some black area in the cropped image. That is because a small part of the detected rectangle is out of the bound of the image. To remedy this, you may pad the image a little bit and do the crop after that. There is an example illustrated in this answer.

Now, we compare the two methods to crop the rotated rectangle from the image.
This method do not require rotating the image and can deal with this problem more elegantly with less code.

This was a very frustrating endeavor, but finally I solved it based on rroowwllaanndd‘s answer. I just had to add the angle correction when the width < height. Without this I got very strange results for images which fulfilled this condition.

def crop_image(rect, image):
    shape = (image.shape[1], image.shape[0])  # cv2.warpAffine expects shape in (length, height)
    center, size, theta = rect
    width, height = tuple(map(int, size))
    center = tuple(map(int, center))
    if width < height:
        theta -= 90
        width, height = height, width

    matrix = cv.getRotationMatrix2D(center=center, angle=theta, scale=1.0)
    image = cv.warpAffine(src=image, M=matrix, dsize=shape)

    x = int(center[0] - width // 2)
    y = int(center[1] - height // 2)

    image = image[y : y + height, x : x + width]

    return image