Справочная информация:
Я пишу программу, которая делает некоторое увеличение данных. Он принимает несколько изображений и делает несколько переворотов, несколько поворотов и гауссов шум. Это делается путем чтения изображений в виде numpy массивов через cv2, а затем я использую memmap для чтения и записи этих изображений в заданном пакете. Каждый раз, когда я готов использовать memmap, я вызываю функцию del.
Проблема:
при запуске программы SIGKILL вызывается через некоторое время. Я смог сузить проблему до того, что при попытке получить доступ к memmap, который был прочитан в память из файла размером 3,6 КБ или больше, вызывается SIGKILL. Поэтому даже при независимом запуске приведенного ниже фрагмента будет одинаковый результат:
source = np.memmap(data.dat, dtype='object', mode='r', shape=(10,))
print(source)
Код:
Обратите внимание, что в коде есть несколько жестких путей.
import numpy as np
import cv2
import os
import random
import imutils
# gets all subdirectories
def getSubdir(path):
subdir = [dI for dI in os.listdir(path) if os.path.isdir(os.path.join(path, dI))]
return subdir
def getFiles(path):
files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
return files
# saving images to main file
def save_array(no_of_images, factor, num_of_elements, arr_path, temp_path, _offset):
i = no_of_images
m = i
k = i
c = 1
source_offset = 0
while (i < (num_of_elements * factor)):
i = m * c
if (i > (num_of_elements * factor)):
i = num_of_elements * factor
k = i % m
temp_source = np.memmap(temp_path, dtype='object', mode='r', shape=(k,), offset=source_offset)
fp = np.memmap(arr_path, dtype='object', mode='r+', shape=(k,), offset=_offset)
fp[:] = temp_source[:]
size_in_bytes = temp_source.size * temp_source.itemsize # temp_source.nbytes
_offset += size_in_bytes
source_offset += size_in_bytes
del temp_source
del fp
c += 1
num_of_elements = num_of_elements * (factor+1)
return (num_of_elements, _offset)
def aug_detection(path):
num_of_rotations = 5
no_of_images = 5 #must always be exactly divisible by 2 and num_of_rotations
for subdir in getSubdir(path):
files = getFiles(path + subdir)
arr = np.empty(len(files), dtype=object)
i = 0
_offset = 0
for file in files:
imagePath = path + subdir + '/' + file
image = cv2.imread(imagePath)
arr[i] = image
i += 1
arr_path = '/home/john/FYP/Dataset/testing2/' + subdir + '/Data/data.dat'
shp = arr.shape
fp = np.memmap(arr_path, dtype='object', mode='w+', shape=shp)
fp[:] = arr[:]
size_in_bytes = fp.size * fp.itemsize
del fp
_offset += size_in_bytes
del arr
temp_path = '/home/john/FYP/Dataset/testing2/' + subdir + '/Data/temp.dat'
num_of_elements = i
del image
del imagePath
# generating rotations
i = no_of_images / num_of_rotations
m = int(i)
k = int(i)
inner_offset = 0
source_offset = 0
c = 1
while (i < num_of_elements):
i = m * c
if (i > num_of_elements):
i = num_of_elements
k = i % m
source = np.memmap(arr_path, dtype='object', mode='r', shape=(k,), offset=source_offset)
size_in_bytes = source.size * source.itemsize # source.nbytes
source_offset += size_in_bytes
arr = np.empty(k * num_of_rotations, dtype=object)
j = 0
for image in source:
rotations = random.sample(range(360), num_of_rotations)
for l in range(num_of_rotations):
rotated = imutils.rotate_bound(image, rotations[l])
arr[j] = rotated
del rotated
j += 1
del source
del image
tmp_shp = arr.shape
if (i == m):
fp = np.memmap(temp_path, dtype='object', mode='w+', shape=tmp_shp)
fp[:] = arr[:]
del arr
size_in_bytes = fp.size * fp.itemsize # arr.nbytes
del fp
else:
fp = np.memmap(temp_path, dtype='object', mode='r+', shape=tmp_shp, offset=inner_offset)
fp[:] = arr[:]
del arr
size_in_bytes = fp.size * fp.itemsize # arr.nbytes
del fp
inner_offset += size_in_bytes
c += 1
# saving rotations to main file
num_of_elements, _offset = save_array(no_of_images, num_of_rotations, num_of_elements, arr_path, temp_path,
_offset)
# generating gaussian noise
i = no_of_images
m = i
k = i
inner_offset = 0
source_offset = 0
c = 1
while (i < num_of_elements):
i = m * c
if (i > num_of_elements):
i = num_of_elements
k = i % m
source = np.memmap(arr_path, dtype='object', mode='r', shape=(k,), offset=source_offset)
size_in_bytes = source.size * source.itemsize # source.nbytes
source_offset += size_in_bytes
arr = np.empty(k, dtype=object)
j = 0
for image in source:
# Generate Gaussian noise
gauss = np.random.normal(0, 1, image.size)
gauss = gauss.reshape(image.shape[0], image.shape[1], image.shape[2]).astype('uint8')
# Add the Gaussian noise to the image
img_gauss = cv2.add(image, gauss)
arr[j] = img_gauss
j += 1
del source
tmp_shp = arr.shape
if (i == m):
fp = np.memmap(temp_path, dtype='object', mode='w+', shape=tmp_shp)
fp[:] = arr[:]
size_in_bytes = fp.size * fp.itemsize # arr.nbytes
inner_offset += size_in_bytes
del fp
else:
fp = np.memmap(temp_path, dtype='object', mode='r+', shape=tmp_shp, offset=inner_offset)
fp[:] = arr[:]
size_in_bytes = fp.size * fp.itemsize # arr.nbytes
inner_offset += size_in_bytes
del fp
c += 1
del gauss
del img_gauss
# saving gaussin noise images to main file
num_of_elements, _offset = save_array(no_of_images, 1, num_of_elements, arr_path, temp_path, _offset)
# generating vertical and horizontal flips
i = no_of_images/2
m = int(i)
k = int(i)
inner_offset = 0
source_offset = 0
c = 1
while (i < num_of_elements):
i = m * c
if (i > num_of_elements):
i = num_of_elements
k = (i % m)
source = np.memmap(arr_path, dtype='object', mode='r', shape=(k,), offset=source_offset)
size_in_bytes = source.size * source.itemsize # source.nbytes
source_offset += size_in_bytes
arr = np.empty((k * 2), dtype=object)
j = 0
for image in source:
flipVertical = cv2.flip(image, 0)
arr[j] = flipVertical
j += 1
flipHorizontal = cv2.flip(image, 1)
arr[j] = flipHorizontal
j += 1
del source
tmp_shp = arr.shape
if (i == m):
fp = np.memmap(temp_path, dtype='object', mode='w+', shape=tmp_shp)
fp[:] = arr[:]
size_in_bytes = fp.size * fp.itemsize # arr.nbytes
del fp
else:
fp = np.memmap(temp_path, dtype='object', mode='r+', shape=tmp_shp, offset=inner_offset)
fp[:] = arr[:]
size_in_bytes = fp.size * fp.itemsize # arr.nbytes
del fp
inner_offset += size_in_bytes
c += 1
del arr
del flipHorizontal
del flipVertical
del image
#saving flips to main file
num_of_elements, _offset = save_array(no_of_images, 2, num_of_elements, arr_path, temp_path, _offset)
aug_detection('/home/john/FYP/Dataset/testing/')