Мы можем использовать sub
для захвата символа, за которым следует число.
df$Category <- sub('([A-Z]\\..*?)\\..*', '\\1', df$Sample)
df
# Sample Gene Exp Category
#1 A.56.100e.ax prk1 3.56 A.56
#2 A.100CAS.500e.er prk1 45.20 A.100CAS
#3 A.300.EFE1.cx prk1 0.21 A.300
#4 A.56.100e.ax csk 51.50 A.56
#5 A.100CAS.500e.er csk 321.47 A.100CAS
#6 A.300.EFE1.cx csk 186.78 A.300
данные
df <- structure(list(Sample = structure(c(3L, 1L, 2L, 3L, 1L, 2L),
.Label = c("A.100CAS.500e.er", "A.300.EFE1.cx", "A.56.100e.ax"),
class = "factor"), Gene = structure(c(2L, 2L, 2L, 1L, 1L, 1L),
.Label = c("csk", "prk1"), class = "factor"),
Exp = c(3.56, 45.2, 0.21, 51.5, 321.47, 186.78)),
class = "data.frame", row.names = c(NA, -6L))