diff options
Diffstat (limited to 'sitemodules/profiles')
8 files changed, 262 insertions, 293 deletions
diff --git a/sitemodules/profiles/files/puppet_server/git-pull-hook b/sitemodules/profiles/files/puppet_server/git-pull-hook deleted file mode 100755 index 0d25a20..0000000 --- a/sitemodules/profiles/files/puppet_server/git-pull-hook +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script takes care of updating the code in a directory by performing -git pull when triggered via an HTTP request to port 8000. - -The script needs the sshpass and git packages installed. - -Configuration is read from /etc/git-pull-hook.ini, ~/.git-pull-hook.ini and -a git-pull-hook.ini in the working directory in that order. -""" - -import logging -import logging.config -import os -import subprocess -from configparser import ConfigParser -from http import HTTPStatus -from http.server import HTTPServer, BaseHTTPRequestHandler - -ENV_FOR_GIT = {"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"} - -TOKENS = [] -GIT_DIRECTORY = "" - -LOGGER = None - - -def read_ini(): - global ENV_FOR_GIT, TOKENS, GIT_DIRECTORY, LOGGER - config = ConfigParser() - config.read( - [ - "/etc/git-pull-hook.ini", - os.path.expanduser("~/.git-pull-hook.ini"), - "git-pull-hook.ini", - ] - ) - ENV_FOR_GIT["SSHPASS"] = config["git-pull-hook"]["ssh_passphrase"] - TOKENS = [token.strip() for token in config["git-pull-hook"]["tokens"].split(",")] - GIT_DIRECTORY = config["git-pull-hook"]["git_directory"] - - logging.config.dictConfig( - { - "version": 1, - "formatters": { - "full": { - "format": "%(asctime)s %(levelname)-8s %(message)s", - "datefmt": "%Y-%m-%d %H:%M:%S", - } - }, - "handlers": { - "file": { - "class": "logging.FileHandler", - "filename": config.get("git-pull-hook", "logfile"), - "formatter": "full", - } - }, - "loggers": {"git-pull-hook": {"handlers": ["file"], "level": "INFO"}}, - } - ) - LOGGER = logging.getLogger("git-pull-hook") - - -class GitHookRequestHandler(BaseHTTPRequestHandler): - """ - Custom HTTP request handler for updating a git repository when called - with a known authentication token in an "Authentication" HTTP header. - """ - - def __init__(self, request, client_address, server): - global LOGGER - self.log = LOGGER - super().__init__(request, client_address, server) - - def _send_data(self, message): - self.send_header("Content-Type", "text/plain; charset=utf8") - self.end_headers() - self.wfile.write(("%s\r\n" % message).encode("UTF-8")) - - def _handle_pull(self): - try: - git_proc = subprocess.run( - ["sshpass", "-e", "-P", "passphrase", "git", "pull"], - env=ENV_FOR_GIT, - cwd=GIT_DIRECTORY, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True, - universal_newlines=True, - ) - for line in git_proc.stdout.splitlines(): - self.log_message("git: %s", line) - except subprocess.CalledProcessError as e: - self.log_error( - "Could not pull changes for %s: %s", GIT_DIRECTORY, e.returncode - ) - for line in e.stdout.splitlines(): - self.log_message("git: %s", line) - self._send_data("Error updating the repository.") - try: - r10k_proc = subprocess.run( - ["/opt/puppetlabs/puppet/bin/r10k", "puppetfile", "install"], - cwd=GIT_DIRECTORY, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True, - universal_newlines=True, - ) - for line in r10k_proc.stdout.splitlines(): - self.log_message("r10k: %s", line) - except subprocess.CalledProcessError as e: - self.log_error("Could not update modules from Puppetfile: %s", e.returncode) - for line in e.stdout.splitlines(): - self.log_message("r10k: %s", line) - self._send_data("Error updating modules.") - - self.send_response(HTTPStatus.OK) - self._send_data("updated %s" % GIT_DIRECTORY) - - # noinspection PyPep8Naming - def do_GET(self): - """ - Handle GET requests, requests to /health are allowed for every caller. - """ - if self.path == "/health": - self.send_response(HTTPStatus.OK) - self._send_data("I'm healthy!") - else: - self.send_error( - HTTPStatus.NOT_FOUND, "You requested something I do not understand." - ) - - # noinspection PyPep8Naming - def do_POST(self): - """ - Handle POST requests requests to / need a valid token in the - "Authentication" HTTP header and trigger a git pull in the configured - directory. - """ - if self.path == "/": - if self.headers["Authentication"] in [token for token in TOKENS]: - self._handle_pull() - else: - self.send_error( - HTTPStatus.UNAUTHORIZED, - 'You have to send a valid token in the "Authentication" header.', - ) - else: - self.send_error( - HTTPStatus.NOT_FOUND, "You requested something I do not understand." - ) - - def log_error(self, format, *args): - self.log.error("%s - %s" % (self.address_string(), format), *args) - - def log_message(self, format, *args): - self.log.info("%s - %s" % (self.address_string(), format), *args) - - -def run(server_class=HTTPServer, handler_class=GitHookRequestHandler): - server_address = ("", 8000) - httpd = server_class(server_address, handler_class) - httpd.serve_forever() - - -if __name__ == "__main__": - read_ini() - run() diff --git a/sitemodules/profiles/files/puppet_server/git-pull-hook.init.sh b/sitemodules/profiles/files/puppet_server/git-pull-hook.init.sh deleted file mode 100644 index 7974a9f..0000000 --- a/sitemodules/profiles/files/puppet_server/git-pull-hook.init.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: git-pull-hook -# Required-Start: $remote_fs -# Required-Stop: $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: CAcert puppet git pull hook -# Description: CAcert puppet git pull hook -### END INIT INFO - -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -BASE=git-pull-hook - -GIT_PULL_HOOK=/usr/local/sbin/git-pull-hook -GIT_PULL_HOOK_PIDFILE=/var/run/$BASE.pid -GIT_PULL_HOOK_LOGFILE=/var/log/$BASE.log -GIT_PULL_HOOK_DESC="Puppet git pull hook" - -# Get lsb functions -. /lib/lsb/init-functions - -# Check git-pull-hook is present -if [ ! -x $GIT_PULL_HOOK ]; then - log_failure_msg "$GIT_PULL_HOOK not present or not executable" - exit 1 -fi - -fail_unless_root() { - if [ "$(id -u)" != '0' ]; then - log_failure_msg "$GIT_PULL_HOOK_DESC must be run as root" - exit 1 - fi -} - -case "$1" in - start) - fail_unless_root - - touch "$GIT_PULL_HOOK_LOGFILE" - chown root:adm "$GIT_PULL_HOOK_LOGFILE" - - log_begin_msg "Starting $GIT_PULL_HOOK_DESC: $GIT_PULL_HOOK" - start-stop-daemon --start --background --no-close \ - --exec "$GIT_PULL_HOOK" \ - --pidfile "$GIT_PULL_HOOK_PIDFILE" \ - --chdir "/" \ - --make-pidfile \ - >> "$GIT_PULL_HOOK_LOGFILE" 2>&1 - log_end_msg $? - ;; - - stop) - fail_unless_root - if [ -f "$GIT_PULL_HOOK_PIDFILE" ]; then - start-stop-daemon --stop --pidfile "$GIT_PULL_HOOK_PIDFILE" --retry 5 - log_end_msg $? - else - log_warning_msg "$GIT_PULL_HOOK_DESC already stopped - file $GIT_PULL_HOOK_PIDFILE not found." - fi - ;; - - restart) - fail_unless_root - git_pull_hook_pid=`cat "$GIT_PULL_HOOK_PIDFILE" 2> /dev/null` - [ -n "$git_pull_hook_pid" ] \ - && ps -p $git_pull_hook_pid > /dev/null 2>&1 \ - && $0 stop - $0 start - ;; - - force-reload) - fail_unless_root - $0 restart - ;; - - status) - status_of_proc -p "$GIT_PULL_HOOK_PIDFILE" "$GIT_PULL_HOOK" "$GIT_PULL_HOOK_DESC" - ;; - - *) - echo "Usage: service git-pull-hook {start|stop|restart|force-reload|status}" - exit 1 - ;; -esac diff --git a/sitemodules/profiles/files/puppet_server/puppet-deploy b/sitemodules/profiles/files/puppet_server/puppet-deploy new file mode 100644 index 0000000..59c1d32 --- /dev/null +++ b/sitemodules/profiles/files/puppet_server/puppet-deploy @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +""" +This script takes care of updating the code in a puppet environment directory by +performing git pull when triggered with the branch ref as argument. The script is +meant to be invoked by webhook. + +The script needs the sshpass and git packages installed. + +Configuration is read from /etc/puppet-deploy.ini, ~/.puppet-deploy.ini and +a puppet-deploy.ini in the working directory in that order. +""" +import argparse +import logging +import logging.config +import os +import subprocess +import sys +from configparser import ConfigParser + +ENV_FOR_GIT = { + "PATH": ":".join( + [ + "/opt/puppetlabs/puppet/bin/", + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin:/bin", + ] + ) +} + +GIT_DIRECTORIES = {} + +LOGGER = None + + +def read_ini(): + global ENV_FOR_GIT, GIT_DIRECTORIES, LOGGER + config = ConfigParser() + config.read( + [ + "/etc/puppet-deploy.ini", + os.path.expanduser("~/.puppet-deploy.ini"), + "puppet-deploy.ini", + ] + ) + if not config.has_section("puppet-deploy"): + logging.error("puppet-deploy section not found in config") + sys.exit(1) + ENV_FOR_GIT["SSHPASS"] = config.get("puppet-deploy", "ssh_passphrase") + branches = config.get("puppet-deploy", "branches").split(",") + for branch in branches: + GIT_DIRECTORIES[branch] = config.get("branch-directories", branch) + + logging.config.dictConfig( + { + "version": 1, + "formatters": { + "full": { + "format": "%(asctime)s %(levelname)-8s %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S", + } + }, + "handlers": { + "file": {"class": "logging.StreamHandler", "formatter": "full"} + }, + "loggers": { + "puppet-deploy": {"handlers": ["file"], "level": "INFO"} + }, + } + ) + LOGGER = logging.getLogger("puppet-deploy") + + +class PuppetDeploy(object): + + def __init__(self, branch_ref): + global LOGGER + self.log = LOGGER + self.branch = branch_ref.split("/")[-1] + super().__init__() + + def handle_pull(self) -> None: + if self.branch not in GIT_DIRECTORIES: + self.log.warn("no directory defined for branch %s", self.branch) + return + work_dir = GIT_DIRECTORIES[self.branch] + + try: + git_proc = subprocess.run( + ["sshpass", "-e", "-P", "passphrase", "git", "pull"], + env=ENV_FOR_GIT, + cwd=work_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + universal_newlines=True, + ) + for line in git_proc.stdout.splitlines(): + self.log.info("git: %s", line) + except subprocess.CalledProcessError as e: + self.log.error( + "could not pull changes for %s: %s", work_dir, e.returncode + ) + for line in e.stdout.splitlines(): + self.log.info("git: %s", line) + self.log.error("error updating the repository in %s", work_dir) + sys.exit(1) + + try: + r10k_proc = subprocess.run( + ["r10k", "puppetfile", "install"], + cwd=work_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + universal_newlines=True, + ) + for line in r10k_proc.stdout.splitlines(): + self.log.info("r10k: %s", line) + except subprocess.CalledProcessError as e: + self.log.error( + "could not update modules from Puppetfile: %s", e.returncode + ) + for line in e.stdout.splitlines(): + self.log.error("r10k: %s", line) + self.log.error("error updating modules") + sys.exit(1) + + print("updated %s" % work_dir) + + +if __name__ == "__main__": + read_ini() + parser = argparse.ArgumentParser(prog="puppet-deploy") + parser.add_argument("ref") + args = parser.parse_args() + PuppetDeploy(args.ref).handle_pull() diff --git a/sitemodules/profiles/files/puppet_server/webhook.service b/sitemodules/profiles/files/puppet_server/webhook.service new file mode 100644 index 0000000..bdcf291 --- /dev/null +++ b/sitemodules/profiles/files/puppet_server/webhook.service @@ -0,0 +1,10 @@ +[Unit] +Description=Small server for creating HTTP endpoints (hooks) +Documentation=https://github.com/adnanh/webhook/ +ConditionPathExists=/etc/webhook.conf + +[Service] +ExecStart=/usr/bin/webhook -nopanic -port 8000 -hotreload -hooks /etc/webhook.conf + +[Install] +WantedBy=multi-user.target diff --git a/sitemodules/profiles/manifests/puppet_server.pp b/sitemodules/profiles/manifests/puppet_server.pp index eb7a4bb..f555193 100644 --- a/sitemodules/profiles/manifests/puppet_server.pp +++ b/sitemodules/profiles/manifests/puppet_server.pp @@ -8,10 +8,13 @@ # # @param git_pull_ssh_passphrase passphrase to use for the ssh key to pull # new code from the control repository -# @param git_pull_directory directory where the puppet control repository -# is checked out -# @param git_pull_tokens list of tokens that are valid to trigger the -# git pull hook +# @param git_pull_branches array of branches to be pulled by the +# puppet-deploy webhook +# @param git_pull_directories branch to directory mapping where the puppet +# control repository for a branch is checked +# out +# @param git_pull_token token that is valid to trigger the +# puppet-deploy webhook # # Examples # -------- @@ -29,59 +32,85 @@ # Copyright # --------- # -# Copyright 2018 Jan Dittberner +# Copyright 2018-2020 Jan Dittberner class profiles::puppet_server ( String $git_pull_ssh_passphrase, - String $git_pull_directory = '/etc/puppetlabs/code/environments/production', - Array[String] $git_pull_tokens, + String $git_pull_token, + Array[String] $git_pull_branches = ["master"], + Hash[String, String] $git_pull_directories = { + 'master' => '/etc/puppetlabs/code/environments/production' + }, ) { - package { 'sshpass': - ensure => installed, - } - - package { 'git': + package { ['git', 'r10k', 'sshpass', 'webhook']: ensure => installed, } file { '/usr/local/sbin/git-pull-hook': - ensure => file, + ensure => absent, + } + file { '/usr/local/sbin/puppet-deploy': owner => 'root', group => 'root', mode => '0750', - source => 'puppet:///modules/profiles/puppet_server/git-pull-hook', - require => [Package['sshpass'], Package['git']], + source => 'puppet:///modules/profiles/puppet_server/puppet-deploy', + require => [Package['sshpass'], Package['git'], Package['r10k'], Package['webhook']], } + service { 'git-pull-hook': + ensure => stopped, + enable => false, + } -> file { '/etc/init.d/git-pull-hook': + ensure => absent, + } + file { '/etc/git-pull-hook.ini': + ensure => absent, + } + + file { '/etc/puppet-deploy.ini': + ensure => file, + owner => 'root', + group => 'root', + mode => '0400', + content => epp( + 'profiles/puppet_server/puppet-deploy.ini.epp', + { + 'ssh_passphrase' => $git_pull_ssh_passphrase, + 'git_branches' => $git_pull_branches, + 'git_directories' => $git_pull_directories, + } + ), + } + file { '/etc/systemd/system/webhook.service': ensure => file, owner => 'root', group => 'root', - mode => '0755', - source => 'puppet:///modules/profiles/puppet_server/git-pull-hook.init.sh' + mode => '0644', + source => 'puppet:///modules/profiles/puppet_server/webhook.service', + } ~> + exec { '/usr/bin/systemctl daemon reload': + refreshonly => true, } - - file { '/etc/git-pull-hook.ini': + file { '/etc/webhook.conf': ensure => file, owner => 'root', group => 'root', mode => '0400', content => epp( - 'profiles/puppet_server/git-pull-hook.ini.epp', - { - 'ssh_passphrase' => $git_pull_ssh_passphrase, - 'tokens' => $git_pull_tokens, - 'git_directory' => $git_pull_directory, + 'profiles/puppet_server/webhook.conf.epp', { + 'token' => $git_pull_token, + 'branches' => $git_pull_branches, } - ) + ), } - service { 'git-pull-hook': + service { 'webhook': ensure => running, enable => true, - subscribe => [File['/etc/git-pull-hook.ini'], File['/usr/local/sbin/git-pull-hook']], + subscribe => [File['/etc/webhook.conf'], File['/usr/local/sbin/puppet-deploy']], require => [ - File['/etc/init.d/git-pull-hook'], File['/usr/local/sbin/git-pull-hook'], - File['/etc/git-pull-hook.ini'], + File['/etc/webhook.conf'], File['/usr/local/sbin/puppet-deploy'], + File['/etc/puppet-deploy.ini'], File['/etc/systemd/system/webhook.service'], ], } }
\ No newline at end of file diff --git a/sitemodules/profiles/templates/puppet_server/git-pull-hook.ini.epp b/sitemodules/profiles/templates/puppet_server/git-pull-hook.ini.epp deleted file mode 100644 index 2876c16..0000000 --- a/sitemodules/profiles/templates/puppet_server/git-pull-hook.ini.epp +++ /dev/null @@ -1,9 +0,0 @@ -<%- | String $ssh_passphrase = undef, String $git_directory = undef, Array[String] $tokens = undef | -%> -# THIS FILE IS MANAGED BY PUPPET, MANUAL CHANGES WILL BE OVERWRITTEN AT THE -# NEXT PUPPET RUN. - -[git-pull-hook] -ssh_passphrase=<%= $ssh_passphrase %> -tokens=<%= $tokens.join(',') %> -git_directory=<%= $git_directory %> -logfile=/var/log/git-pull-hook.log diff --git a/sitemodules/profiles/templates/puppet_server/puppet-deploy.ini.epp b/sitemodules/profiles/templates/puppet_server/puppet-deploy.ini.epp new file mode 100644 index 0000000..fb23fea --- /dev/null +++ b/sitemodules/profiles/templates/puppet_server/puppet-deploy.ini.epp @@ -0,0 +1,14 @@ +<%- | +String $ssh_passphrase = undef, Array[String] $git_branches = undef, Hash[String, String] $git_directories = undef +| -%> +# THIS FILE IS MANAGED BY PUPPET, MANUAL CHANGES WILL BE OVERWRITTEN AT THE +# NEXT PUPPET RUN. + +[puppet-deploy] +ssh_passphrase=<%= $ssh_passphrase %> +branches=<%= $git_branches.join(",") %> + +[branch-directories] +<% $git_directories.each |String $branch, String $directory| { -%> +<%= $branch %>=<%= $directory %> +<% } -%>
\ No newline at end of file diff --git a/sitemodules/profiles/templates/puppet_server/webhook.conf.epp b/sitemodules/profiles/templates/puppet_server/webhook.conf.epp new file mode 100644 index 0000000..6db1062 --- /dev/null +++ b/sitemodules/profiles/templates/puppet_server/webhook.conf.epp @@ -0,0 +1,40 @@ +<%- | String $token = undef, Array[String] $branches = undef | -%> +[ + { + "id": "puppet-deploy", + "execute-command": "/usr/local/sbin/puppet-deploy", + "command-working-directory": "/etc/puppetlabs/code/environments", + "include-command-output-in-response": true, + "include-command-output-in-response-on-error": true, + "pass-arguments-to-command": [ + { + "source": "payload", + "name": "ref" + } + ], + "trigger-rule": { + "and": [ + { + "match": { + "type": "value", + "value": "<%= $token %>", + "parameter": { + "source": "header", + "name": "Authentication" + } + } + }, + { + "match": { + "type": "regex", + "regex": "^refs/heads/(<%= $branches.join("|") %>)$", + "parameter": { + "source": "payload", + "name": "ref" + } + } + } + ] + } + } +]
\ No newline at end of file |