Этот кусок кода должен сделать работу за вас:
public static CommandResult exeMapReduceAlikeGroupCmd(DB db, String colName, String keyFunc, DBObject query,
String reduceFunc, DBObject initial) {
BasicDBObject cmd = new BasicDBObject("ns", colName);// this is the collection name
cmd.append("$keyf", keyFunc);// keyf function
cmd.append("cond", query); // cond query object
cmd.append("$reduce", reduceFunc);// reduce function
cmd.append("initial", initial);// initial value
cmd = new BasicDBObject("group", cmd);
return db.command(cmd, db.getOptions());
}