Image Pyramid. Having trouble creating the desired composite image
Question:
What I am trying to do:
Combine these two images:
Using this mask:
to create this output:
The assignment:
Write a program to create a composite image of the two images with the mask, based on image pyramids.
Now, This is what I have tried so far:
import cv2
import numpy as np
# Read the input images and the mask
image1 = cv2.imread("figure2-assignment3.jpg")
image2 = cv2.imread("figure3-assignment3.jpg")
mask = cv2.imread("figure4-assignment3.jpg", cv2.IMREAD_GRAYSCALE)
# Smooth out the mask
mask = cv2.GaussianBlur(mask, (5, 5), 0)
# Convert mask to float32 and normalize to range [0, 1]
mask = mask.astype(np.float32) / 255.0
# Duplicate the mask to match the number of channels in the images
mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
# Generate Gaussian pyramids for both images and the mask
gaussian_pyramid_image1 = [image1]
gaussian_pyramid_image2 = [image2]
gaussian_pyramid_mask = [mask]
for _ in range(6):
image1 = cv2.pyrDown(image1)
gaussian_pyramid_image1.append(image1)
image2 = cv2.pyrDown(image2)
gaussian_pyramid_image2.append(image2)
mask = cv2.pyrDown(mask)
gaussian_pyramid_mask.append(mask)
# Generate Laplacian pyramids for both images
laplacian_pyramid_image1 = [gaussian_pyramid_image1[-1]]
laplacian_pyramid_image2 = [gaussian_pyramid_image2[-1]]
for i in range(5, 0, -1): # Start from the second last level
image1_up = cv2.pyrUp(gaussian_pyramid_image1[i])
image2_up = cv2.pyrUp(gaussian_pyramid_image2[i])
image1_resized = cv2.resize(gaussian_pyramid_image1[i - 1], (image1_up.shape[1], image1_up.shape[0]))
image2_resized = cv2.resize(gaussian_pyramid_image2[i - 1], (image2_up.shape[1], image2_up.shape[0]))
laplacian_image1 = cv2.subtract(image1_resized, image1_up)
laplacian_image2 = cv2.subtract(image2_resized, image2_up)
laplacian_pyramid_image1.append(laplacian_image1)
laplacian_pyramid_image2.append(laplacian_image2)
# Generate Gaussian pyramid for the mask
gaussian_pyramid_mask = [gaussian_pyramid_mask[-1]]
# Start from the second last level
for i in range(5, 0, -1):
mask_up = cv2.pyrUp(gaussian_pyramid_mask[-1])
mask_resized = cv2.resize(gaussian_pyramid_mask[-1], (mask_up.shape[1], mask_up.shape[0]))
gaussian_pyramid_mask.append(mask_resized)
# Combine the corresponding levels of Laplacian pyramids using the mask
composite_pyramid = []
for img1, img2, msk in zip(laplacian_pyramid_image1, laplacian_pyramid_image2, gaussian_pyramid_mask):
img1_resized = cv2.resize(img1, (msk.shape[1], msk.shape[0]))
img2_resized = cv2.resize(img2, (msk.shape[1], msk.shape[0]))
composite_level = img1_resized * msk + img2_resized * (1.0 - msk)
composite_pyramid.append(composite_level)
# Collapse the composite pyramid to obtain the composite image
composite_image = composite_pyramid[-1]
for i in range(len(composite_pyramid) - 2, -1, -1):
composite_image_up = cv2.pyrUp(composite_image)
composite_image_resized = cv2.resize(composite_pyramid[i], (composite_image_up.shape[1],
composite_image_up.shape[0]))
composite_image = cv2.add(composite_image_resized, composite_image_up)
# Save the composite image
cv2.imwrite("composite_image_2.jpg", composite_image)
And this is the best I could produce:
Now what am I possibly doing wrong? I can get the hand, but the right side of the composite image is not the correct one.
Answers:
APPROACH 1: Custom Alpha Blending (shorter)
I would not worry that much about using either Gaussian or Laplacian pyramids for this. Instead, you can perform alpha blending with a smoothed-out version of the mask provided (this ensures smooth borders) to arrive to the desired output. Here is my approach to solving your problem:
import cv2
import numpy as np
# Read the input images and the mask
mask = cv2.imread("mask.jpeg", cv2.IMREAD_GRAYSCALE)
image1 = cv2.imread("image-1.jpeg")
image2 = cv2.imread("image-2.jpeg")
# Resize images to match mask dimensions
height, width = mask.shape[:2]
image1 = cv2.resize(image1, (width, height))
image2 = cv2.resize(image2, (width, height))
# Smooth out the mask and normalize to range [0, 1]
transparency_gradient = cv2.blur(mask, (25, 25))
transparency_gradient = cv2.cvtColor(transparency_gradient, cv2.COLOR_GRAY2BGR)
transparency_gradient = transparency_gradient / 255.0 # Normalize to range [0, 1]
# Perform manual alpha blending with transparency gradient
composite_image = image1 * transparency_gradient + image2 * (1 - transparency_gradient)
# Save the result
cv2.imwrite("composite_image.png", composite_image)
Which yields this final image:
APPROACH 2: Blending through Gaussian and Laplacian Pyramids
If the objective is to achieve such blending through pyramids, below is a code that does the trick:
import cv2
import numpy as np
# Read the input images and the mask
mask = cv2.imread("mask.jpeg", cv2.IMREAD_GRAYSCALE)
image1 = cv2.imread("image-1.jpeg")
image2 = cv2.imread("image-2.jpeg")
# Set the level of the pyramids (tweak it for better accuracy)
levels=6
# Resize images to match mask dimensions
height, width = image1.shape[:2]
mask = cv2.resize(mask, (width, height), interpolation=cv2.INTER_LINEAR)
# Duplicate the mask to match the number of channels in the images
mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
# Generate Gaussian pyramids for both images and the mask
gaussian_pyramid1 = [image1.astype(np.float32)]
gaussian_pyramid2 = [image2.astype(np.float32)]
mask_pyramid = [mask.astype(np.float32) / 255.0]
for _ in range(levels - 1):
image1 = cv2.pyrDown(image1)
image2 = cv2.pyrDown(image2)
mask = cv2.pyrDown(mask)
gaussian_pyramid1.append(image1.astype(np.float32))
gaussian_pyramid2.append(image2.astype(np.float32))
mask_pyramid.append(mask.astype(np.float32) / 255.0)
# Generate Laplacian pyramids for both images
laplacian_pyramid1 = [gaussian_pyramid1[levels - 1]]
laplacian_pyramid2 = [gaussian_pyramid2[levels - 1]]
for i in range(levels - 2, -1, -1):
expanded1 = cv2.pyrUp(gaussian_pyramid1[i + 1], dstsize=(gaussian_pyramid1[i].shape[1], gaussian_pyramid1[i].shape[0]))
expanded2 = cv2.pyrUp(gaussian_pyramid2[i + 1], dstsize=(gaussian_pyramid2[i].shape[1], gaussian_pyramid2[i].shape[0]))
laplacian1 = cv2.subtract(gaussian_pyramid1[i], expanded1)
laplacian2 = cv2.subtract(gaussian_pyramid2[i], expanded2)
laplacian_pyramid1.append(laplacian1)
laplacian_pyramid2.append(laplacian2)
# Combine the corresponding levels of Laplacian pyramids using the mask
composite_pyramid = []
for laplacian1, laplacian2, mask in zip(laplacian_pyramid1, laplacian_pyramid2, mask_pyramid):
mask_resized = cv2.resize(mask, (laplacian1.shape[1], laplacian1.shape[0]), interpolation=cv2.INTER_LINEAR)
composite_level = laplacian1 * mask_resized + laplacian2 * (1.0 - mask_resized)
composite_pyramid.append(composite_level)
# Reconstruct the final blended image
composite_image = composite_pyramid[0]
for i in range(1, levels):
composite_image = cv2.pyrUp(composite_image, dstsize=(composite_pyramid[i].shape[1], composite_pyramid[i].shape[0]))
composite_image += composite_pyramid[i]
# Ensure pixel values are within valid range
composite_image = np.clip(composite_image, 0, 255).astype(np.uint8)
# Save Image
cv2.imwrite("composite_image.png", composite_image)
Which produces:
It’s your choice now… Good luck! And may the code be with you…
What I am trying to do:
Combine these two images:
Using this mask:
to create this output:
The assignment:
Write a program to create a composite image of the two images with the mask, based on image pyramids.
Now, This is what I have tried so far:
import cv2
import numpy as np
# Read the input images and the mask
image1 = cv2.imread("figure2-assignment3.jpg")
image2 = cv2.imread("figure3-assignment3.jpg")
mask = cv2.imread("figure4-assignment3.jpg", cv2.IMREAD_GRAYSCALE)
# Smooth out the mask
mask = cv2.GaussianBlur(mask, (5, 5), 0)
# Convert mask to float32 and normalize to range [0, 1]
mask = mask.astype(np.float32) / 255.0
# Duplicate the mask to match the number of channels in the images
mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
# Generate Gaussian pyramids for both images and the mask
gaussian_pyramid_image1 = [image1]
gaussian_pyramid_image2 = [image2]
gaussian_pyramid_mask = [mask]
for _ in range(6):
image1 = cv2.pyrDown(image1)
gaussian_pyramid_image1.append(image1)
image2 = cv2.pyrDown(image2)
gaussian_pyramid_image2.append(image2)
mask = cv2.pyrDown(mask)
gaussian_pyramid_mask.append(mask)
# Generate Laplacian pyramids for both images
laplacian_pyramid_image1 = [gaussian_pyramid_image1[-1]]
laplacian_pyramid_image2 = [gaussian_pyramid_image2[-1]]
for i in range(5, 0, -1): # Start from the second last level
image1_up = cv2.pyrUp(gaussian_pyramid_image1[i])
image2_up = cv2.pyrUp(gaussian_pyramid_image2[i])
image1_resized = cv2.resize(gaussian_pyramid_image1[i - 1], (image1_up.shape[1], image1_up.shape[0]))
image2_resized = cv2.resize(gaussian_pyramid_image2[i - 1], (image2_up.shape[1], image2_up.shape[0]))
laplacian_image1 = cv2.subtract(image1_resized, image1_up)
laplacian_image2 = cv2.subtract(image2_resized, image2_up)
laplacian_pyramid_image1.append(laplacian_image1)
laplacian_pyramid_image2.append(laplacian_image2)
# Generate Gaussian pyramid for the mask
gaussian_pyramid_mask = [gaussian_pyramid_mask[-1]]
# Start from the second last level
for i in range(5, 0, -1):
mask_up = cv2.pyrUp(gaussian_pyramid_mask[-1])
mask_resized = cv2.resize(gaussian_pyramid_mask[-1], (mask_up.shape[1], mask_up.shape[0]))
gaussian_pyramid_mask.append(mask_resized)
# Combine the corresponding levels of Laplacian pyramids using the mask
composite_pyramid = []
for img1, img2, msk in zip(laplacian_pyramid_image1, laplacian_pyramid_image2, gaussian_pyramid_mask):
img1_resized = cv2.resize(img1, (msk.shape[1], msk.shape[0]))
img2_resized = cv2.resize(img2, (msk.shape[1], msk.shape[0]))
composite_level = img1_resized * msk + img2_resized * (1.0 - msk)
composite_pyramid.append(composite_level)
# Collapse the composite pyramid to obtain the composite image
composite_image = composite_pyramid[-1]
for i in range(len(composite_pyramid) - 2, -1, -1):
composite_image_up = cv2.pyrUp(composite_image)
composite_image_resized = cv2.resize(composite_pyramid[i], (composite_image_up.shape[1],
composite_image_up.shape[0]))
composite_image = cv2.add(composite_image_resized, composite_image_up)
# Save the composite image
cv2.imwrite("composite_image_2.jpg", composite_image)
And this is the best I could produce:
Now what am I possibly doing wrong? I can get the hand, but the right side of the composite image is not the correct one.
APPROACH 1: Custom Alpha Blending (shorter)
I would not worry that much about using either Gaussian or Laplacian pyramids for this. Instead, you can perform alpha blending with a smoothed-out version of the mask provided (this ensures smooth borders) to arrive to the desired output. Here is my approach to solving your problem:
import cv2
import numpy as np
# Read the input images and the mask
mask = cv2.imread("mask.jpeg", cv2.IMREAD_GRAYSCALE)
image1 = cv2.imread("image-1.jpeg")
image2 = cv2.imread("image-2.jpeg")
# Resize images to match mask dimensions
height, width = mask.shape[:2]
image1 = cv2.resize(image1, (width, height))
image2 = cv2.resize(image2, (width, height))
# Smooth out the mask and normalize to range [0, 1]
transparency_gradient = cv2.blur(mask, (25, 25))
transparency_gradient = cv2.cvtColor(transparency_gradient, cv2.COLOR_GRAY2BGR)
transparency_gradient = transparency_gradient / 255.0 # Normalize to range [0, 1]
# Perform manual alpha blending with transparency gradient
composite_image = image1 * transparency_gradient + image2 * (1 - transparency_gradient)
# Save the result
cv2.imwrite("composite_image.png", composite_image)
Which yields this final image:
APPROACH 2: Blending through Gaussian and Laplacian Pyramids
If the objective is to achieve such blending through pyramids, below is a code that does the trick:
import cv2
import numpy as np
# Read the input images and the mask
mask = cv2.imread("mask.jpeg", cv2.IMREAD_GRAYSCALE)
image1 = cv2.imread("image-1.jpeg")
image2 = cv2.imread("image-2.jpeg")
# Set the level of the pyramids (tweak it for better accuracy)
levels=6
# Resize images to match mask dimensions
height, width = image1.shape[:2]
mask = cv2.resize(mask, (width, height), interpolation=cv2.INTER_LINEAR)
# Duplicate the mask to match the number of channels in the images
mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
# Generate Gaussian pyramids for both images and the mask
gaussian_pyramid1 = [image1.astype(np.float32)]
gaussian_pyramid2 = [image2.astype(np.float32)]
mask_pyramid = [mask.astype(np.float32) / 255.0]
for _ in range(levels - 1):
image1 = cv2.pyrDown(image1)
image2 = cv2.pyrDown(image2)
mask = cv2.pyrDown(mask)
gaussian_pyramid1.append(image1.astype(np.float32))
gaussian_pyramid2.append(image2.astype(np.float32))
mask_pyramid.append(mask.astype(np.float32) / 255.0)
# Generate Laplacian pyramids for both images
laplacian_pyramid1 = [gaussian_pyramid1[levels - 1]]
laplacian_pyramid2 = [gaussian_pyramid2[levels - 1]]
for i in range(levels - 2, -1, -1):
expanded1 = cv2.pyrUp(gaussian_pyramid1[i + 1], dstsize=(gaussian_pyramid1[i].shape[1], gaussian_pyramid1[i].shape[0]))
expanded2 = cv2.pyrUp(gaussian_pyramid2[i + 1], dstsize=(gaussian_pyramid2[i].shape[1], gaussian_pyramid2[i].shape[0]))
laplacian1 = cv2.subtract(gaussian_pyramid1[i], expanded1)
laplacian2 = cv2.subtract(gaussian_pyramid2[i], expanded2)
laplacian_pyramid1.append(laplacian1)
laplacian_pyramid2.append(laplacian2)
# Combine the corresponding levels of Laplacian pyramids using the mask
composite_pyramid = []
for laplacian1, laplacian2, mask in zip(laplacian_pyramid1, laplacian_pyramid2, mask_pyramid):
mask_resized = cv2.resize(mask, (laplacian1.shape[1], laplacian1.shape[0]), interpolation=cv2.INTER_LINEAR)
composite_level = laplacian1 * mask_resized + laplacian2 * (1.0 - mask_resized)
composite_pyramid.append(composite_level)
# Reconstruct the final blended image
composite_image = composite_pyramid[0]
for i in range(1, levels):
composite_image = cv2.pyrUp(composite_image, dstsize=(composite_pyramid[i].shape[1], composite_pyramid[i].shape[0]))
composite_image += composite_pyramid[i]
# Ensure pixel values are within valid range
composite_image = np.clip(composite_image, 0, 255).astype(np.uint8)
# Save Image
cv2.imwrite("composite_image.png", composite_image)
Which produces:
It’s your choice now… Good luck! And may the code be with you…