Это самое простое решение, которое я нашел:
void
evaluatePixel()
{
float4 color = sampleNearest(src,outCoord());
float maxR = max(minColor.r, maxColor.r);
float maxG = max(minColor.g, maxColor.g);
float maxB = max(minColor.b, maxColor.b);
float minR = min(minColor.r, maxColor.r);
float minG = min(minColor.g, maxColor.g);
float minB = min(minColor.b, maxColor.b);
dst = color;
// Check whether a color is within the range
if(color.r >= minR && color.g >= minG && color.b >= minB)
{
if(color.r <= maxR && color.g <= maxG && color.b <= maxB)
{
dst = float4(0.0,0.0,0.0,0.0);
}
}
}