Для этого вы можете использовать оконные функции:
select brand, car_model, cnt_car
from (select c.*, dense_rank() over (order by cnt_brand, brand) as seqnum_b
from (select brand, car_model, count(*) as cnt_car,
row_number() over (partition by brand order by count(*) desc) as seqnum_bc,
sum(count(*)) over (partition by brand) as cnt_brand
from cars c
group by brand, car_model
) c
) c
where seqnum_bc <= 5 and seqnum_b <= 3
order by cnt_brand desc, brand, cnt desc;
Если вы знаете, что у каждой марки (или, по крайней мере, у каждой марки) есть как минимум пять автомобилей, вы можете упростить запрос до:
select brand, car_model, cnt_car
from (select brand, car_model, count(*) as cnt_car,
row_number() over (partition by brand order by count(*) desc) as seqnum_bc,
sum(count(*)) over (partition by brand) as cnt_brand
from cars c
group by brand, car_model
) c
where seqnum_bc <= 5
order by cnt_brand desc, brand, cnt desc
limit 15