У меня проблемы с запуском скрипта bash при добавлении его в пользовательские данные в шаблоне запуска ec2. Я посмотрел на предложения и попробовал несколько подходов, включая предложение AWS, используя MIME multi-part в сценарии. Я попробовал директиву # cloud-boothook, но она запускает только часть моего скрипта при загрузке. Интересно, что после загрузки экземпляра я могу успешно запустить скрипт, вызвав его вручную в / var / lib / cloud / instances / instance id / user-data.txt. Я не уверен, что еще попробовать, поэтому любая помощь приветствуется. Ниже мой сценарий.
#cloud-boothook
#!/bin/bash
apt-get update
pip install -I ansible==2.6.2
cd /tmp
wget https://dl.google.com/go/go1.11.linux-amd64.tar.gz
tar -xvf go1.11.linux-amd64.tar.gz
mv go /usr/lib
apt-get -y install checkinstall build-essential
apt-get -y install libreadline6-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev
cd /tmp
wget https://www.python.org/ftp/python/2.7.15/Python-2.7.15.tgz
tar -xvf Python-2.7.15.tgz
cd Python-2.7.15
./configure --prefix=/opt/python27myapp --enable-shared --enable-optimizations LDFLAGS=-Wl,-rpath=/opt/python27myapp/lib
make
checkinstall --pkgname=python27myapp --pkgversion=2.7.15 --pkgrelease=0 --nodoc --default make install
cd /tmp
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
/opt/python27myapp/bin/python /tmp/get-pip.py
/opt/python27myapp/bin/pip install --progress-bar=off virtualenv
set -euo pipefail
DEPLOYER="myapp"
PYTHON_DIST_PACKAGES="/usr/lib/python2.7/dist-packages"
PYTHON_SITE_PACKAGES="lib/python2.7/site-packages"
ANSIBLE_VENV_PATH="/mnt/ansible-12f"
ANSIBLE_USER="ansible"
ANSIBLE_VERSION="2.6.2.0"
ANSIBLE_USER_HOME="/home/${ANSIBLE_USER}"
TF_PLAYBOOK_REPO="git@github.com:myorg/${DEPLOYER}.git"
TF_PLAYBOOK_GITREF="2019.8.0"
TF_PLAYBOOK_OPTIONS="" # perhaps -vvvv
TF_PLAYBOOK_PATH="playbooks/twelve_factor/deploy.yml"
TF_APP_CONFIG_JSON="extra-vars.json"
TF_SCRATCH_DIR=$(mktemp -d -t tmp.XXXXXXXXXX)
TF_APP_CONFIG_PATH="${TF_SCRATCH_DIR}/config"
TF_ENVIRONMENT=""
EC2_INSTANCE_TAGS=""
CONFIG_BUCKET="databag-versions"
REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -c -r .region)
app_user="myapp"
git-with-ssm-key()
{
ssm_key="git_checkout_key"; shift
ssh-agent bash -o pipefail -c '
if aws ssm get-parameters \
--region "'$REGION'" \
--names "'$ssm_key'" \
--with-decryption \
--output text \
--query Parameters[0].Value |
ssh-add -k -
then
git "$@"
else
echo >&2 "ERROR: Failed to get or add key: '$ssm_key'"
exit 1
fi
' bash "$@"
}
#ssh-keyscan github.com >> ~/.ssh/known_hosts
# ============================================================================
cleanup() {
rm -rf $TF_SCRATCH_DIR
}
final_steps() {
cleanup
}
trap final_steps EXIT
# ============================================================================
install_packages() {
apt-get -y install jq
}
check_for_aws_cli() {
if ! aws help 1> /dev/null 2>&1; then
apt-get -y install awscli
fi
if ! aws help 1> /dev/null 2>&1; then
echo "The aws cli is not installed." 1>&2
exit 1
fi
echo "Found: $(aws --version 2>&1)"
}
application_deployed() {
[ -e "$TF_APP_HOME/current" ];
}
set_tf_app_home() {
TF_APP_HOME="$(jq .deploy_env.deploy_to $TF_APP_CONFIG_PATH | sed -e 's/^"//' -e 's/"$//')"
}
set_ec2_instance_tags() {
# We grab the EC2 tags of this instance.
instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
filters="Name=resource-id,Values=${instance_id}"
EC2_INSTANCE_TAGS=$(aws ec2 describe-tags --region $REGION --filters "${filters}" | jq .Tags)
}
set_tf_environment() {
# The tag whose Key is "Name" has the Value we want. Strip leading/trailing quotes.
name=$(echo "$EC2_INSTANCE_TAGS" | jq '.[] | select(.Key == "Environment") | .Value')
TF_ENVIRONMENT=$(echo $name | sed -e 's/^"//' -e 's/"$//')
}
set_config_bucket() {
case "$TF_ENVIRONMENT" in
innovate|production|bolt|operations|blackhat)
CONFIG_BUCKET="databag-versions"
;;
*)
CONFIG_BUCKET="databag-dev"
;;
esac
}
retrieve_configuration_source() {
# The tag whose Key is "Name" has the Value we want. Strip leading/trailing quotes.
selectName='.[] | select(.Key == "Name") | .Value'
name=$(echo "$EC2_INSTANCE_TAGS" | jq "$selectName" | sed -e 's/^"//' -e 's/"$//')
s3key="databags/$(echo $name | sed -e 's;-;/;g')"
aws s3 cp s3://${CONFIG_BUCKET}/${s3key} ${TF_APP_CONFIG_PATH}
set_git_ssh_key
set_tf_app_home
}
install_python() {
apt-get -y install python
pip install virtualenv
virtualenv $ANSIBLE_VENV_PATH
}
install_ansible() {
source ${ANSIBLE_VENV_PATH}/bin/activate
pip install ansible==${ANSIBLE_VERSION}
echo "$PYTHON_DIST_PACKAGES" > "${ANSIBLE_VENV_PATH}/${PYTHON_SITE_PACKAGES}/dist-packages.pth"
# This will go wrong if the system python path changes.
if [ ! -d "$PYTHON_DIST_PACKAGES" ]; then
echo "ERROR: the system python packages location does not exist: $PYTHON_DIST_PACKAGES"
exit 1
fi
# Having established a link between our vitualenv and the system python, we
# can now install python-apt.
pip install python-apt
}
add_playbook_user() {
if ! getent group $ANSIBLE_USER > /dev/null 2>&1; then
addgroup --system $ANSIBLE_USER
fi
if ! id -u $ANSIBLE_USER > /dev/null 2>&1; then
adduser --system --home $ANSIBLE_USER_HOME --shell /bin/false \
--ingroup $ANSIBLE_USER --disabled-password \
--gecos GECOS \
$ANSIBLE_USER
fi
if [ ! -d "$ANSIBLE_USER_HOME/.ssh" ]; then
mkdir $ANSIBLE_USER_HOME/.ssh
fi
chown ${ANSIBLE_USER}:${ANSIBLE_USER} $ANSIBLE_USER_HOME/.ssh
chmod 700 $ANSIBLE_USER_HOME/.ssh
echo "StrictHostKeyChecking no" > $ANSIBLE_USER_HOME/.ssh/config
chown ${ANSIBLE_USER}:${ANSIBLE_USER} $ANSIBLE_USER_HOME/.ssh/config
# echo ${GIT_SSH_KEY} > $ANSIBLE_USER_HOME/.ssh/id_rsa
echo $GIT_SSH_KEY | sed -e "s/-----BEGIN RSA PRIVATE KEY-----/&\n/" -e "s/-----END RSA PRIVATE KEY-----/\n&/" -e "s/\S\{64\}/&\n/g" > $ANSIBLE_USER_HOME/.ssh/id_rsa
chown ${ANSIBLE_USER}:${ANSIBLE_USER} $ANSIBLE_USER_HOME/.ssh/id_rsa
chmod 600 $ANSIBLE_USER_HOME/.ssh/id_rsa
if ! getent group $app_user > /dev/null 2>&1; then
addgroup --system $app_user
fi
if ! id -u $app_user > /dev/null 2>&1; then
adduser --system --home ${ANSIBLE_USER_HOME}/myapp --shell /bin/false \
--ingroup $app_user --disabled-password \
--gecos GECOS \
$app_user
fi
}
retrieve_playbook() {
rm -rf "${ANSIBLE_USER_HOME}/${DEPLOYER}"
(
cd "${ANSIBLE_USER_HOME}"
git-with-ssm-key /githubsshkeys/gitreader clone --branch "$TF_PLAYBOOK_GITREF" "$TF_PLAYBOOK_REPO"
)
chown -R ansible:ansible "${ANSIBLE_USER_HOME}/${DEPLOYER}"
}
patch_playbooks() {
awk '/^- name:/ {f=0} /^- name: Establish SSH credentials/ {f=1} !f;' ${ANSIBLE_USER_HOME}/myapp/playbooks/twelve_factor/app_user.yml > /tmp/temp_app_user.yml
cp /tmp/temp_app_user.yml ${ANSIBLE_USER_HOME}/myapp/playbooks/twelve_factor/app_user.yml # temp file is necessary since awk can't edit in-place
# remove the 'singleton' run operation... this isn't a singleton, and the playbook fails on the check to determine that.
sed -i 's/^.*singleton.*$//' ${ANSIBLE_USER_HOME}/myapp/playbooks/twelve_factor/app.yml
# fix the "invalid ioctl" warning, which is non-breaking but creates ugly warnings in the log
sed -i -e 's/mesg n .*true/tty -s \&\& mesg n/g' /root/.profile
# fix myapp user permissions in the /mnt/ansible-12f directories
chown -R myapp:myapp ${ANSIBLE_VENV_PATH}
# set up proper git SSH access for the ansible user
echo -e "host github.com\n HostName github.com\n IdentityFile ~/.ssh/id_rsa\n User git" >> $ANSIBLE_USER_HOME/.ssh/config
# set up the same git access for the myapp user
if [ ! -d "/mnt/myapp/.ssh" ]; then
mkdir -p /mnt/myapp/.ssh
fi
# ensure the directory will have the right permissions
mkdir -p /mnt/myapp/releases
cp ${ANSIBLE_USER_HOME}/.ssh/* /mnt/myapp/.ssh
chown -R ${app_user}:${app_user} /mnt/myapp
}
set_git_ssh_key() {
GIT_SSH_KEY="$(aws ssm get-parameters --region $REGION --names git_checkout_key --with-decryption --query Parameters[0].Value --output text)"
ssh-keyscan github.com >> ~/.ssh/known_hosts
}
write_inventory() {
IP_ADDRESS=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
cat - <<END_INVENTORY | sed -e 's/^ *//' > "${ANSIBLE_USER_HOME}/inventory"
[${IP_ADDRESS}]
${IP_ADDRESS} ansible_python_interpreter=${ANSIBLE_VENV_PATH}/bin/python
[linux]
${IP_ADDRESS}
# This bizarre group name will never be used anywhere.
# We need another group with an entry in it to avoid triggering
# the cmd_singleton section.
[12345_xx_%%%%_xxxx_]
10.10.10.10
END_INVENTORY
}
# Located in the directory in which ansible-playbook is executed; it is
# automatically picked up as Ansible runs.
write_ansible_settings() {
cat - <<END_SETTINGS | sed -e 's/^ *//' > "${ANSIBLE_USER_HOME}/ansible.cfg"
[defaults]
inventory = /etc/ansible/hosts
library = /usr/local/opt/ansible/libexec/lib/python2.7/site-packages/ansible/modules
remote_tmp = /tmp/.ansible-${USER}/tmp
pattern = *
forks = 5
poll_interval = 15
transport = smart
gathering = implicit
host_key_checking = False
# SSH timeout
timeout = 30
# we ssh as user medidata, becoming the 12-factor user, so:
# see https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user
allow_world_readable_tmpfiles = True
ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host}
action_plugins = /usr/share/ansible_plugins/action_plugins
callback_plugins = /usr/share/ansible_plugins/callback_plugins
connection_plugins = /usr/share/ansible_plugins/connection_plugins
lookup_plugins = /usr/share/ansible_plugins/lookup_plugins
vars_plugins = /usr/share/ansible_plugins/vars_plugins
filter_plugins = /usr/share/ansible_plugins/filter_plugins
fact_caching = memory
retry_files_enabled = False
[ssh_connection]
# We have sometimes an error raised: Timeout (32s) waiting for privilege escalation prompt
# To avoid this, we make multiple attempts at the connection:
retries = 5
ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o ServerAliveInterval=120
control_path = %(directory)s/%%h-%%r
pipelining = False
END_SETTINGS
}
write_deployment_config() {
cat - <<END_SETTINGS > "${ANSIBLE_USER_HOME}/${TF_APP_CONFIG_JSON}"
END_SETTINGS
}
run_deployment() {
write_inventory
write_ansible_settings
write_deployment_config
(
cd ${ANSIBLE_USER_HOME}
ansible-playbook "${DEPLOYER}/${TF_PLAYBOOK_PATH}" \
-i ${ANSIBLE_USER_HOME}/inventory \
"${DEPLOYER}/${TF_PLAYBOOK_PATH}" \
--connection=local \
--extra-vars @${TF_APP_CONFIG_PATH}
2> /tmp/ansible.err | tee /tmp/ansible.out
)
}
# -----------------------------------------------------
# ---------------- Script Starts Here -----------------
# -----------------------------------------------------
install_packages # do we need to check if packages already installed?
check_for_aws_cli
set_ec2_instance_tags
set_tf_environment
retrieve_configuration_source
if application_deployed; then
echo "Application already deployed; taking no action"
else
add_playbook_user
install_python
install_ansible
retrieve_playbook
patch_playbooks
run_deployment
fi
chown -R ${app_user}:${app_user} /mnt/myapp/services
chown -R ${app_user}:${app_user} /etc/sv/myapp*