Использовать понимание вложенного списка:
df = pd.DataFrame({'rnti':['0x00000034,0x0000008a,0x00000145','0x00000034,0x0000008a']})
print (df)
rnti
0 0x00000034,0x0000008a,0x00000145
1 0x00000034,0x0000008a
rnti_list = [[int(idx, 16) for idx in x.split(",")] for x in df['rnti']]
print (rnti_list)
[[52, 138, 325], [52, 138]]
df['new'] = [[int(idx, 16) for idx in x.split(",")] for x in df['rnti']]
print (df)
rnti new
0 0x00000034,0x0000008a,0x00000145 [52, 138, 325]
1 0x00000034,0x0000008a [52, 138]