сделать что-то вроде:
var $table = $("#tableId"),
$window = $(window);
$table.dialog({
// other properties set here ...
maxHeight: $window.height() - 50,
maxWidth: $window.width() - 50,
height: $table.outerHeight(true) + 50,
width: $table.outerWidth(true) + 50
});
// I always give some room, hence the 50