Если SQL Server 2008, вы можете сделать это прямо через CONVERT
declare @hexstring varchar(max);
set @hexstring = 'abcedf012439';
/*SQL Server 2005 Specific*/
select cast('' as xml).value('xs:hexBinary( substring(sql:variable("@hexstring"), sql:column("t.pos")) )', 'varbinary(max)')
from (select case substring(@hexstring, 1, 2) when '0x' then 3 else 0 end) as t(pos)
/*SQL Server 2008 Specific*/
set @hexstring = 'abcedf012439';
select CONVERT(varbinary(max), @hexstring, 2);
set @hexstring = '0xabcedf012439';
select CONVERT(varbinary(max), @hexstring, 1);
Источник: http://blogs.msdn.com/b/sqltips/archive/2008/07/02/converting-from-hex-string-to-varbinary-and-vice-versa.aspx