См. Комментарий uvgroovy выше. Вам нужно 2 нуля в конце поля 'dir'.
int silently_remove_directory(LPCTSTR dir) // Fully qualified name of the directory being deleted, without trailing backslash
{
int len = strlen(dir) + 2; // required to set 2 nulls at end of argument to SHFileOperation.
char* tempdir = (char*) malloc(len);
memset(tempdir,0,len);
strcpy(tempdir,dir);
SHFILEOPSTRUCT file_op = {
NULL,
FO_DELETE,
tempdir,
"",
FOF_NOCONFIRMATION |
FOF_NOERRORUI |
FOF_SILENT,
false,
0,
"" };
int ret = SHFileOperation(&file_op);
free(tempdir);
return ret; // returns 0 on success, non zero on failure.
}