CV2 SeamlessClone (-215: утверждение не выполнено)

Question

У меня есть нейронная сеть, которая выводит сегменты лица - я работаю над функцией, которая объединяет эти сегменты вместе, а затем клонирует их в реальное лицо.

Примеры изображений здесь: https://imgur.com/a/HnpqhEE, у меня нет репутации, чтобы включить их в строку.

Пока что моя функция берет макияж лица и губы и объединяет их с дополнением. Левый и правый глаза затем клонируются с помощью seamlessClone (правый глаз сначала переворачивается). Затем комбинированные сегменты макияжа клонируются в нормальное лицо.

Очень редко моя комбинационная функция завершается с ошибкой и возвращает (-215: подтверждение не выполнено) 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows в функции 'cv :: Mat :: Mat' ошибка. </p>

Моя функция ниже, я только видел ошибку в последнем бесшовном клоне

def combineFace(images, radius = 70):
    # Given image segments and eye radii, combine face.
    realFace    = tensor2im(images['realNormal'])
    makeupFace  = tensor2im(images['fakeMakeupFace'])
    makeupLeft  = tensor2im(images['fakeMakeupLeft'])
    makeupRight = tensor2im(images['fakeMakeupRight'])
    makeupLips  = tensor2im(images['fakeMakeupLips'])
    makeupRight = cv2.flip(makeupRight, 1)
    # I use cv2 and dlib to get face landmarks and interesting points.
    normalLandmarks = faceLandmarks(realFace)
    facePoints      = getFacePoints(normalLandmarks)
    # PP means pupil points
    outerPoints, leftPP, rightPP, lipPoints, eyeMids = facePoints
    # eyeMid is (x, y) of center of eye obtained from landmark points
    leftEye  = eyeMids[0]
    rightEye = eyeMids[1]

    faceMask = np.zeros(realFace.shape, realFace.dtype)
    cv2.fillPoly(faceMask, [outerPoints], [255, 255, 255])
    cv2.fillPoly(faceMask, [lipPoints], [0, 0, 0])
    cv2.fillPoly(faceMask, [leftPP], [0, 0, 0])
    cv2.fillPoly(faceMask, [rightPP], [0, 0, 0])


    # Occasionally, the eye segments overlap eachother so I cut the right eye from the left and vice 
    # versa

    leftMask = np.zeros(realFace.shape, realFace.dtype)
    cv2.circle(leftMask, leftEye, radius, [255, 255, 255], -1)
    cv2.circle(leftMask, rightEye, radius,  [0, 0, 0], -1)
    # Errors if i do not use UMat
    cv2.circle(cv2.UMat(makeupLeft), rightEye, radius, [0, 0, 0], -1)

    rightMask = np.zeros(realFace.shape, realFace.dtype)
    cv2.circle(rightMask, rightEye, radius, [255, 255, 255], -1)
    cv2.circle(rightMask, leftEye, radius,  [0, 0, 0], -1)
    cv2.circle(cv2.UMat(makeupRight), leftEye, radius, [0, 0, 0], -1)

    # Combine face output and lips
    baseCombine = makeupFace + makeupLips
    # Left Eye
    output = cv2.seamlessClone(makeupLeft,  baseCombine, leftMask, leftEye, cv2.MIXED_CLONE)
    output = cv2.seamlessClone(makeupRight, output, rightMask, rightEye, cv2.MIXED_CLONE)


    # Get center of face
    faceRect = cv2.boundingRect(outerPoints)
    x, y, w, h = faceRect

    output = cv2.bitwise_and(output, faceMask)
    center = ( x + w // 2, y + h // 2)

    # I have only seen the function error at this point 
    combinedFace = cv2.seamlessClone(output, realFace, faceMask, center, cv2.MIXED_CLONE)
    return combinedFace

Есть идеи, почему это иногда приводит к ошибкам?

Все входные изображения имеют вид (256, 256, 3)

Answer 1

Эта версия функции работает намного лучше. Что-то не так с моим расчетом центра лица, что вызывало ошибку

def combineFace(images, radius = 70):
    # Given image segments and eye radii, combine face.
    realFace    = tensor2im(images['realNormal'])
    makeupFace  = tensor2im(images['fakeMakeupFace'])
    makeupLeft  = tensor2im(images['fakeMakeupLeft'])
    makeupRight = tensor2im(images['fakeMakeupRight'])
    makeupLips  = tensor2im(images['fakeMakeupLips'])

    # Right eye is flipped before input into the network.
    makeupRight = cv2.flip(makeupRight, 1)

    normalLandmarks = faceLandmarks(realFace)
    facePoints      = getFacePoints(normalLandmarks)

    outerPoints, leftPP, rightPP, lipPoints, eyeMids = facePoints
    leftEye  = eyeMids[0]
    rightEye = eyeMids[1]


    leftMask = np.zeros(makeupLeft.shape, makeupLeft.dtype)
    cv2.circle(leftMask, leftEye, radius,   [255, 255, 255], -1)
    cv2.circle(leftMask, rightEye, radius,  [0, 0, 0], -1)
    # Errors if i do not use cv2.UMat
    cv2.circle(cv2.UMat(makeupLeft), rightEye, radius, [0, 0, 0], -1)

    rightMask = np.zeros(makeupRight.shape, makeupRight.dtype)
    cv2.circle(rightMask, rightEye, radius, [255, 255, 255], -1)
    cv2.circle(rightMask, leftEye, radius,  [0, 0, 0], -1)
    cv2.circle(cv2.UMat(makeupRight), leftEye, radius, [0, 0, 0], -1)

    # Base output is combination of face without lips and pupils + lips
    baseCombine = makeupFace + makeupLips
    # Areas around eyes are changes
    output = cv2.seamlessClone(makeupLeft,  baseCombine, leftMask,  leftEye, cv2.MIXED_CLONE)
    output = cv2.seamlessClone(makeupRight, output,      rightMask, rightEye, cv2.MIXED_CLONE)

    # Find center of face 
    faceRect   = cv2.boundingRect(outerPoints)

    x, y, w, h = faceRect 

    if x < 0:
        x = 0
    if y < 0: 
        y = 0

    faceCenter = ( x + w // 2, y + h // 2)

    croppedOutput = output[y:y+h, x:x+w]

    faceMask = np.zeros(realFace.shape, realFace.dtype)
    cv2.fillPoly(faceMask, [outerPoints], [255, 255, 255])
    cv2.fillPoly(faceMask, [lipPoints],   [0, 0, 0])
    cv2.fillPoly(faceMask, [leftPP],      [0, 0, 0])
    cv2.fillPoly(faceMask, [rightPP],     [0, 0, 0])

    croppedMask = faceMask[y:y+h, x:x+w]

    if len(croppedOutput) == 0:
        print("OUTPUT 0")
        print("FACE RECT: ", faceRect)


    sourceW, sourceH, sCH = realFace.shape
    width, height, ch = croppedOutput.shape

    faceWidth  = width/2
    faceHeight = height/2
    xdiff = 0
    ydiff = 0
    cx = faceCenter[0]
    cy = faceCenter[1]

    if cx - faceWidth < 0:
        # Face overflows left
        xdiff = abs(cx - faceWidth)
    if cx + faceWidth > sourceW:
        xdiff = (cx + faceWidth - sourceW) * -1

    if cy + faceHeight > sourceH:
        ydiff = (cy + faceHeight - sourceH) * -1
    if cy - faceHeight < 0:
        ydiff = abs(cy - faceHeight)

    centerx = int(cx + xdiff)
    centery = int(cy + ydiff)
    center = (centerx, centery)

    # We move center, also move mask?


    combinedFace = cv2.seamlessClone(croppedOutput, realFace, croppedMask, center, cv2.MIXED_CLONE)
    return combinedFace