Выберите:
$ sed 's/(\([^)]*\))/,\1/' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
$ sed 's/(/,/; s/)//' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
.
$ awk '{sub(/\(/,","); sub(/\)/,"")} 1' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
$ awk 'match($0,/\([^)]*\)/){$0= substr($0,1,RSTART-1) "," substr($0,RSTART+1,RLENGTH-2) substr($0,RSTART+RLENGTH) } 1' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
$ awk 'BEGIN{FS=OFS=","} split($1,a,/[()]/) > 1{$1=a[1] "," a[2]} 1' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
$ gawk '{$0=gensub(/\(([^)]*)\)/,",\\1",1)} 1' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
$ gawk 'match($0,/([^(]*)\(([^)]*)\)(.*)/,a){$0=a[1] "," a[2] a[3]} 1' file
Col1,col2,col3,col4,col5
a,23,12,test(1),test2
b,30,15,test1(2),test3
Последние 2 требуют GNU awk для gensub () и 3-го аргумента для match () соответственно. Есть и альтернативы.