Group multiple barcodes on a single sticker together from an image with multiple stickers using python

Ok since pyzbar/zbar seems to have bugs that cause its bounding boxes to catch multiple codes, or not detect codes that are rotated too much, I’ll use OpenCV’s barcode detection, rectify the codes, then use pyzbar for decoding. OpenCV can also decode, but not as many different types.

Approach:

• find codes, get bounding boxes
• for each bounding box, enlarge it width-wise and see what other codes it overlaps with
• build "groups"

Input:

Detect barcodes with OpenCV:

det = cv.barcode.BarcodeDetector()
(rv, detections) = det.detect(im)
# detections: four corner points per detection

Extract rectangle:

def extract_region_from_corners(image, corners):
    # order:
    #   [1] top left     [2] top right
    #   [0] bottom left  [3] bottom right
    (bl, tl, tr, br) = corners

    # axis vectors
    vx = tr - tl
    vy = bl - tl

    lx = np.linalg.norm(vx)
    ly = np.linalg.norm(vy)

    H = np.eye(3)
    H[0:2,2] = tl # origin
    H[:2,0] = vx / lx
    H[:2,1] = vy / ly

    dst = cv.warpAffine(src=image,
        M=H[:2], dsize=(int(lx), int(ly)),
        flags=cv.INTER_LINEAR | cv.WARP_INVERSE_MAP)

    return dst

Utility function:

def corners_to_rrect(corners):
    # order:
    #   [1] top left     [2] top right
    #   [0] bottom left  [3] bottom right
    (bl, tl, tr, br) = corners

    vx = ((tr - tl) + (br - bl)) / 2
    vy = ((bl - tl) + (br - tr)) / 2
    lx = np.linalg.norm(vx)
    ly = np.linalg.norm(vy)

    center = tuple(corners.mean(axis=0))
    size = (lx, ly)
    angle = np.arctan2(vx[1], vx[0]) / np.pi * 180 # degrees

    return (center, size, angle)

Extract codes, decode, note their RotatedRect positions:

found_codes = []
canvas = im.copy()

for detection_corners in detections:
    rrect = corners_to_rrect(detection_corners)

    (rrect_width, rrect_height) = rrect[1]
    assert rrect_width > rrect_height, ("assuming barcode lies lengthwise", rrect)

    roi = extract_region_from_corners(image=im, corners=detection_corners)

    [code] = pyzbar.decode(roi, symbols=[ZBarSymbol.CODE128, ZBarSymbol.EAN13])
    print(code.type, code.data, rrect)

    found_codes.append( (rrect, code) )

    cv.polylines(img=canvas, pts=[detection_corners.astype(np.int32)], isClosed=True, color=(255, 0, 0), thickness=2)

CODE128 b'07FFD58D47189879' ((706.9937, 355.28094), (434.7604, 65.09412), 15.141749040805594)
CODE128 b'07FFD58D47189878' ((266.48895, 361.89154), (435.78812, 65.95062), -15.051276355059604)
CODE128 b'07FFD58D47189876' ((237.65492, 816.5005), (434.7883, 65.28357), 15.058296081979087)
CODE128 b'07FFD58D47189877' ((731.69257, 817.5774), (435.56052, 62.905884), -15.084296904602034)
EAN13 b'0871828002084' ((228.3433, 239.54503), (235.90378, 66.31835), -15.219580753945182)
EAN13 b'0871828002077' ((705.7166, 693.0964), (236.39447, 65.9507), -15.102472037983436)
EAN13 b'0871828002091' ((742.64703, 237.18982), (240.23358, 67.790794), 15.171352788215723)
EAN13 b'0871828002060' ((270.11478, 696.054), (236.27463, 64.16398), 15.201185346963047)

More utility functions:

def enlarge_rrect(rrect, factor=1, fx=1, fy=1):
    (center, size, angle) = rrect
    (width, height) = size
    new_size = (width * factor * fx, height * factor * fy)
    return (center, new_size, angle)

def merge_intersecting_sets(sets):
    # sets = set(map(frozenset, sets))
    while True:
        oldcount = len(sets)

        # merge or add
        newsets = set()
        for thisset in sets:
            for thatset in newsets:
                if thisset & thatset:
                    newsets.remove(thatset)
                    newsets.add(thisset | thatset)
                    break
            else:
                newsets.add(thisset)

        sets = newsets
        if len(sets) == oldcount:
            break
    
    return sets

# assert merge_intersecting_sets([{1,2}, {2,3}, {3,4}, {5,6}]) == {frozenset({1,2,3,4}), frozenset({5,6})}

_{Note: This set operation has no effect on this data because the data is simple enough. Theoretically, it is required. Say you have three codes A,B,C beside each other, where A,B are adjacent and B,C are adjacent but A,C are not adjacent. This operation merges the sets {A,B} and {B,C} into {A,B,C}.}

Determine groups using enlarged RotatedRect and intersection test:

def associate_rrects(rrects, fx=1, fy=1):
    "associate RotatedRect instances, given enlargement factors in horizontal and vertical direction"

    # build connected components by adjacency
    components = set()
    for (i, thisrect) in enumerate(rrects):
        thisenlarged = enlarge_rrect(thisrect, fx=fx, fy=fy)

        component = {i}
        for (j, thatrect) in enumerate(rrects):
            (rv, intersection) = cv.rotatedRectangleIntersection(thisenlarged, thatrect)
            if rv != cv.INTERSECT_NONE: # i.e. INTERSECT_PARTIAL, INTERSECT_FULL
                component.add(j)

        components.add(frozenset(component))
    
    # merge intersecting components (transitivitiy)
    components = merge_intersecting_sets(components)

    return components


components = associate_rrects([rrect for rrect, code in found_codes], fy=5)
print(components)

{frozenset({1, 4}), frozenset({2, 7}), frozenset({0, 6}), frozenset({3, 5})}

Now you can pick from found_codes using those indices.

Drawing the groups, using convex hull:

canvas = im.copy()
for component in components:
    component_codes = [found_codes[i] for i in component]

    component_corners = np.concatenate([
        cv.boxPoints(rrect)
        for (rrect, code) in component_codes
    ])
    hull = cv.convexHull(component_corners)
    cv.polylines(img=canvas, pts=[hull.astype(np.int32)], isClosed=True, color=(255, 255, 0), thickness=2)

    for (rrect, code) in component_codes:
        #print(rrect, code)
        cv.polylines(img=canvas, pts=[cv.boxPoints(rrect).astype(int)], isClosed=True, color=(255, 0, 0), thickness=2)
        cv.putText(canvas, text=str(code.data),
            org=np.int0(rrect[0]), fontFace=cv.FONT_HERSHEY_SIMPLEX,
            fontScale=0.7, color=(0,0,0), thickness=8)
        cv.putText(canvas, text=str(code.data),
            org=np.int0(rrect[0]), fontFace=cv.FONT_HERSHEY_SIMPLEX,
            fontScale=0.7, color=(0,255,255), thickness=2)

Entire thing: https://gist.github.com/crackwitz/3a7e7e5d698274198393737415ef409a