Как насчет использования внутреннего соединения:
int[] numbers = new[] { 2, 3, 4, 5, 6 };
from r in dtlRecord
join number in numbers on r.Number equals number
group r by r.Number into g
select new {
Number = g.Key,
Sum = g.Sum(r => r.Count)
};
или
dtlRecord
.Join(numbers, r => r.Number, number => number, (r, number) => new {r, number})
.GroupBy(arg => arg.r.Number, arg => arg.r)
.Select(g => new
{
Number = g.Key,
Sum = g.Sum(r => r.Count)
});