Если я вас правильно понимаю:
tab <- table(x, y)
rfreq <- rowSums(tab)/sum(tab)
cfreq <- colSums(tab)/sum(tab)
# exclude all rows containing less than 5% of the data
tab[rfreq >= 0.05, ]
# exclude all columns less than 5%
tab[, cfreq >= 0.05]
# exclude both rows and columns
tab[rfreq >= 0.05, cfreq >= 0.05]