Вы можете использовать группу по
select channelid, curveid,min(abs(xvalue - $myvalue))
from table
WHERE xvalue >= ($myvalue + .5) AND xvalue <= ($myvalue - .5)
group by channelid, curveid
ORDER by min(abs(xvalue - $myvalue)) asc
и, если вам нужно, также xvalue
select table.*, t.min_diff from table
inner join (
select channelid, curveid,min(abs(xvalue - $myvalue)) min_diff
from table
WHERE xvalue >= ($myvalue + .5) AND xvalue <= ($myvalue - .5)
group by channelid, curveid
ORDER by min(abs(xvalue - $myvalue)) asc
) t on t.channelid = table.channelid
and y.curveid = table.curveid
and abs(table.xvalue - $myvalue) = t.min_diff