Вы можете сделать:
import pandas as pd
data1 = [['A', 1],
['B', 2],
['C', 3],
['D', 4]]
data2 = [['A', 1],
['A', 3],
['B', 4],
['B', 3],
['B', 7],
['C', 4],
['D', 6],
['D', 8]]
df1 = pd.DataFrame(data=data1, columns=['c1', 'c2'])
df2 = pd.DataFrame(data=data2, columns=['c1', 'c2'])
result = pd.concat([df1, df2]).groupby('c1')['c2'].apply(list)
print(result)
Вывод
c1
A [1, 1, 3]
B [2, 4, 3, 7]
C [3, 4]
D [4, 6, 8]
Name: c2, dtype: object
Или без столбцов:
df1 = pd.DataFrame(data=data1)
df2 = pd.DataFrame(data=data2)
result = pd.concat([df1, df2]).groupby(0)[1].apply(list)
print(result)
Вывод
0
A [1, 1, 3]
B [2, 4, 3, 7]
C [3, 4]
D [4, 6, 8]
Name: 1, dtype: object