python многопроцессорный пул импорта, cpu_count: вызывает вечный цикл - PullRequest
0 голосов
/ 18 июня 2020

Код, использующий многопроцессорность, вызывает навсегда l oop.

Я использую сборку системы распознавания радужной оболочки глаза. это функция matching. все работает нормально, пока не будет multiprocessing деталь.

Я прилагаю снимок экрана с выводом ошибки ниже, чтобы вы получили лучшее представление.

RuntimeError:
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.

Код:

##-----------------------------------------------------------------------------
##  Import
##-----------------------------------------------------------------------------
import numpy as np
from os import listdir
from fnmatch import filter
import scipy.io as sio
from multiprocessing import Pool, cpu_count
from itertools import repeat

import warnings
warnings.filterwarnings("ignore")


##-----------------------------------------------------------------------------
##  Function
##-----------------------------------------------------------------------------
def matching(template_extr, mask_extr, temp_dir, threshold=0.38):
    """
    Description:
        Match the extracted template with database.

    Input:
        template_extr   - Extracted template.
        mask_extr       - Extracted mask.
        threshold       - Threshold of distance.
        temp_dir        - Directory contains templates.

    Output:
        List of strings of matched files, 0 if not, -1 if no registered sample.
    """
    # Get the number of accounts in the database
    n_files = len(filter(listdir(temp_dir), '*.mat'))
    if n_files == 0:
        return -1

    # Use all cores to calculate Hamming distances
    args = zip(
        sorted(listdir(temp_dir)),
        repeat(template_extr),
        repeat(mask_extr),
        repeat(temp_dir),
    )
    with Pool(processes=cpu_count()) as pools:
        result_list = pools.starmap(matchingPool, args)

    filenames = [result_list[i][0] for i in range(len(result_list))]
    hm_dists = np.array([result_list[i][1] for i in range(len(result_list))])

    # Remove NaN elements
    ind_valid = np.where(hm_dists>0)[0]
    hm_dists = hm_dists[ind_valid]
    filenames = [filenames[idx] for idx in ind_valid]

    # Threshold and give the result ID
    ind_thres = np.where(hm_dists<=threshold)[0]

    # Return
    if len(ind_thres)==0:
        return 0
    else:
        hm_dists = hm_dists[ind_thres]
        filenames = [filenames[idx] for idx in ind_thres]
        ind_sort = np.argsort(hm_dists)
        return [filenames[idx] for idx in ind_sort]


#------------------------------------------------------------------------------
def calHammingDist(template1, mask1, template2, mask2):
    """
    Description:
        Calculate the Hamming distance between two iris templates.

    Input:
        template1   - The first template.
        mask1       - The first noise mask.
        template2   - The second template.
        mask2       - The second noise mask.

    Output:
        hd          - The Hamming distance as a ratio.
    """
    # Initialize
    hd = np.nan

    # Shift template left and right, use the lowest Hamming distance
    for shifts in range(-8,9):
        template1s = shiftbits(template1, shifts)
        mask1s = shiftbits(mask1, shifts)

        mask = np.logical_or(mask1s, mask2)
        nummaskbits = np.sum(mask==1)
        totalbits = template1s.size - nummaskbits

        C = np.logical_xor(template1s, template2)
        C = np.logical_and(C, np.logical_not(mask))
        bitsdiff = np.sum(C==1)

        if totalbits==0:
            hd = np.nan
        else:
            hd1 = bitsdiff / totalbits
            if hd1 < hd or np.isnan(hd):
                hd = hd1

    # Return
    return hd


#------------------------------------------------------------------------------
def shiftbits(template, noshifts):
    """
    Description:
        Shift the bit-wise iris patterns.

    Input:
        template    - The template to be shifted.
        noshifts    - The number of shift operators, positive for right
                      direction and negative for left direction.

    Output:
        templatenew - The shifted template.
    """
    # Initialize
    templatenew = np.zeros(template.shape)
    width = template.shape[1]
    s = 2 * np.abs(noshifts)
    p = width - s

    # Shift
    if noshifts == 0:
        templatenew = template

    elif noshifts < 0:
        x = np.arange(p)
        templatenew[:, x] = template[:, s + x]
        x = np.arange(p, width)
        templatenew[:, x] = template[:, x - p]

    else:
        x = np.arange(s, width)
        templatenew[:, x] = template[:, x - s]
        x = np.arange(s)
        templatenew[:, x] = template[:, p + x]

    # Return
    return templatenew


#------------------------------------------------------------------------------
def matchingPool(file_temp_name, template_extr, mask_extr, temp_dir):
    """
    Description:
        Perform matching session within a Pool of parallel computation

    Input:
        file_temp_name  - File name of the examining template
        template_extr   - Extracted template
        mask_extr       - Extracted mask of noise

    Output:
        hm_dist         - Hamming distance
    """
    # Load each account
    data_template = sio.loadmat('%s%s'% (temp_dir, file_temp_name))
    template = data_template['template']
    mask = data_template['mask']

    # Calculate the Hamming distance
    hm_dist = calHammingDist(template_extr, mask_extr, template, mask)
    return (file_temp_name, hm_dist)

как мне убрать многопроцессорность и заставить код работать нормально?

ссылка на Dropbox скриншоты

1 Ответ

0 голосов
/ 05 июля 2020

Используйте python s itertools.starmap()

надеюсь, что это поможет

...