Используйте initWithCustomView и установите вместо него UIButton в качестве customView:
UIButton *subBtn = [UIButton buttonWithType:UIButtonTypeCustom];
subBtn.frame = CGRectMake(0, 0, 50, 40); //adjust as needed
[subBtn setImage:[UIImage imageNamed:@"mapbutton.png"] forState:UIControlStateNormal];
[subBtn addTarget:self action:@selector(btnMapClicked:) forControlEvents:UIControlEventTouchUpInside];
UIBarButtonItem *btnMap = [[UIBarButtonItem alloc] initWithCustomView:subBtn];
self.navigationItem.rightBarButtonItem = btnMap;
[btnMap release];
Обратите внимание, что отправителем в btnMapClicked: теперь будет UIButton вместо UIBarButtonItem.