Replace custom Python webhook with packaged webhook

This commit replaces the custom Python webhook for puppet environment
deployment with the go based webhook from the Debian package with the
same name.

The puppet-deploy script only takes care of pulling from git and running
r10k now. The web requests are now handled by webhook.

8 files changed, 262 insertions, 293 deletions

diff --git a/sitemodules/profiles/files/puppet_server/git-pull-hook b/sitemodules/profiles/files/puppet_server/git-pull-hook
deleted file mode 100755
index 0d25a20..0000000
--- a/sitemodules/profiles/files/puppet_server/git-pull-hook
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-This script takes care of updating the code in a directory by performing
-git pull when triggered via an HTTP request to port 8000.
-
-The script needs the sshpass and git packages installed.
-
-Configuration is read from /etc/git-pull-hook.ini, ~/.git-pull-hook.ini and
-a git-pull-hook.ini in the working directory in that order.
-"""
-
-import logging
-import logging.config
-import os
-import subprocess
-from configparser import ConfigParser
-from http import HTTPStatus
-from http.server import HTTPServer, BaseHTTPRequestHandler
-
-ENV_FOR_GIT = {"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
-
-TOKENS = []
-GIT_DIRECTORY = ""
-
-LOGGER = None
-
-
-def read_ini():
-    global ENV_FOR_GIT, TOKENS, GIT_DIRECTORY, LOGGER
-    config = ConfigParser()
-    config.read(
-        [
-            "/etc/git-pull-hook.ini",
-            os.path.expanduser("~/.git-pull-hook.ini"),
-            "git-pull-hook.ini",
-        ]
-    )
-    ENV_FOR_GIT["SSHPASS"] = config["git-pull-hook"]["ssh_passphrase"]
-    TOKENS = [token.strip() for token in config["git-pull-hook"]["tokens"].split(",")]
-    GIT_DIRECTORY = config["git-pull-hook"]["git_directory"]
-
-    logging.config.dictConfig(
-        {
-            "version": 1,
-            "formatters": {
-                "full": {
-                    "format": "%(asctime)s %(levelname)-8s %(message)s",
-                    "datefmt": "%Y-%m-%d %H:%M:%S",
-                }
-            },
-            "handlers": {
-                "file": {
-                    "class": "logging.FileHandler",
-                    "filename": config.get("git-pull-hook", "logfile"),
-                    "formatter": "full",
-                }
-            },
-            "loggers": {"git-pull-hook": {"handlers": ["file"], "level": "INFO"}},
-        }
-    )
-    LOGGER = logging.getLogger("git-pull-hook")
-
-
-class GitHookRequestHandler(BaseHTTPRequestHandler):
-    """
-    Custom HTTP request handler for updating a git repository when called
-    with a known authentication token in an "Authentication" HTTP header.
-    """
-
-    def __init__(self, request, client_address, server):
-        global LOGGER
-        self.log = LOGGER
-        super().__init__(request, client_address, server)
-
-    def _send_data(self, message):
-        self.send_header("Content-Type", "text/plain; charset=utf8")
-        self.end_headers()
-        self.wfile.write(("%s\r\n" % message).encode("UTF-8"))
-
-    def _handle_pull(self):
-        try:
-            git_proc = subprocess.run(
-                ["sshpass", "-e", "-P", "passphrase", "git", "pull"],
-                env=ENV_FOR_GIT,
-                cwd=GIT_DIRECTORY,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                check=True,
-                universal_newlines=True,
-            )
-            for line in git_proc.stdout.splitlines():
-                self.log_message("git: %s", line)
-        except subprocess.CalledProcessError as e:
-            self.log_error(
-                "Could not pull changes for %s: %s", GIT_DIRECTORY, e.returncode
-            )
-            for line in e.stdout.splitlines():
-                self.log_message("git: %s", line)
-            self._send_data("Error updating the repository.")
-        try:
-            r10k_proc = subprocess.run(
-                ["/opt/puppetlabs/puppet/bin/r10k", "puppetfile", "install"],
-                cwd=GIT_DIRECTORY,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                check=True,
-                universal_newlines=True,
-            )
-            for line in r10k_proc.stdout.splitlines():
-                self.log_message("r10k: %s", line)
-        except subprocess.CalledProcessError as e:
-            self.log_error("Could not update modules from Puppetfile: %s", e.returncode)
-            for line in e.stdout.splitlines():
-                self.log_message("r10k: %s", line)
-            self._send_data("Error updating modules.")
-
-        self.send_response(HTTPStatus.OK)
-        self._send_data("updated %s" % GIT_DIRECTORY)
-
-    # noinspection PyPep8Naming
-    def do_GET(self):
-        """
-        Handle GET requests, requests to /health are allowed for every caller.
-        """
-        if self.path == "/health":
-            self.send_response(HTTPStatus.OK)
-            self._send_data("I'm healthy!")
-        else:
-            self.send_error(
-                HTTPStatus.NOT_FOUND, "You requested something I do not understand."
-            )
-
-    # noinspection PyPep8Naming
-    def do_POST(self):
-        """
-        Handle POST requests requests to / need a valid token in the
-        "Authentication" HTTP header and trigger a git pull in the configured
-        directory.
-        """
-        if self.path == "/":
-            if self.headers["Authentication"] in [token for token in TOKENS]:
-                self._handle_pull()
-            else:
-                self.send_error(
-                    HTTPStatus.UNAUTHORIZED,
-                    'You have to send a valid token in the "Authentication" header.',
-                )
-        else:
-            self.send_error(
-                HTTPStatus.NOT_FOUND, "You requested something I do not understand."
-            )
-
-    def log_error(self, format, *args):
-        self.log.error("%s - %s" % (self.address_string(), format), *args)
-
-    def log_message(self, format, *args):
-        self.log.info("%s - %s" % (self.address_string(), format), *args)
-
-
-def run(server_class=HTTPServer, handler_class=GitHookRequestHandler):
-    server_address = ("", 8000)
-    httpd = server_class(server_address, handler_class)
-    httpd.serve_forever()
-
-
-if __name__ == "__main__":
-    read_ini()
-    run()
diff --git a/sitemodules/profiles/files/puppet_server/git-pull-hook.init.sh b/sitemodules/profiles/files/puppet_server/git-pull-hook.init.sh
deleted file mode 100644
index 7974a9f..0000000
--- a/sitemodules/profiles/files/puppet_server/git-pull-hook.init.sh
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/bin/sh
-### BEGIN INIT INFO
-# Provides:          git-pull-hook
-# Required-Start:    $remote_fs
-# Required-Stop:     $remote_fs
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: CAcert puppet git pull hook
-# Description:       CAcert puppet git pull hook
-### END INIT INFO
-
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-
-BASE=git-pull-hook
-
-GIT_PULL_HOOK=/usr/local/sbin/git-pull-hook
-GIT_PULL_HOOK_PIDFILE=/var/run/$BASE.pid
-GIT_PULL_HOOK_LOGFILE=/var/log/$BASE.log
-GIT_PULL_HOOK_DESC="Puppet git pull hook"
-
-# Get lsb functions
-. /lib/lsb/init-functions
-
-# Check git-pull-hook is present
-if [ ! -x $GIT_PULL_HOOK ]; then
-  log_failure_msg "$GIT_PULL_HOOK not present or not executable"
-  exit 1
-fi
-
-fail_unless_root() {
-  if [ "$(id -u)" != '0' ]; then
-    log_failure_msg "$GIT_PULL_HOOK_DESC must be run as root"
-    exit 1
-  fi
-}
-
-case "$1" in
-  start)
-    fail_unless_root
-
-    touch "$GIT_PULL_HOOK_LOGFILE"
-    chown root:adm "$GIT_PULL_HOOK_LOGFILE"
-
-    log_begin_msg "Starting $GIT_PULL_HOOK_DESC: $GIT_PULL_HOOK"
-    start-stop-daemon --start --background --no-close \
-      --exec "$GIT_PULL_HOOK" \
-      --pidfile "$GIT_PULL_HOOK_PIDFILE" \
-      --chdir "/" \
-      --make-pidfile \
-         >> "$GIT_PULL_HOOK_LOGFILE" 2>&1
-    log_end_msg $?
-    ;;
-
-  stop)
-    fail_unless_root
-    if [ -f "$GIT_PULL_HOOK_PIDFILE" ]; then
-      start-stop-daemon --stop --pidfile "$GIT_PULL_HOOK_PIDFILE" --retry 5
-      log_end_msg $?
-    else
-      log_warning_msg "$GIT_PULL_HOOK_DESC already stopped - file $GIT_PULL_HOOK_PIDFILE not found."
-    fi
-    ;;
-
-  restart)
-    fail_unless_root
-  git_pull_hook_pid=`cat "$GIT_PULL_HOOK_PIDFILE" 2> /dev/null`
-  [ -n "$git_pull_hook_pid" ] \
-    && ps -p $git_pull_hook_pid > /dev/null 2>&1 \
-    && $0 stop
-  $0 start
-  ;;
-
-  force-reload)
-    fail_unless_root
-    $0 restart
-    ;;
-
-  status)
-    status_of_proc -p "$GIT_PULL_HOOK_PIDFILE" "$GIT_PULL_HOOK" "$GIT_PULL_HOOK_DESC"
-    ;;
-
-  *)
-    echo "Usage: service git-pull-hook {start|stop|restart|force-reload|status}"
-    exit 1
-    ;;
-esac
diff --git a/sitemodules/profiles/files/puppet_server/puppet-deploy b/sitemodules/profiles/files/puppet_server/puppet-deploy
new file mode 100644
index 0000000..59c1d32
--- /dev/null
+++ b/sitemodules/profiles/files/puppet_server/puppet-deploy
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+"""
+This script takes care of updating the code in a puppet environment directory by
+performing git pull when triggered with the branch ref as argument. The script is
+meant to be invoked by webhook.
+
+The script needs the sshpass and git packages installed.
+
+Configuration is read from /etc/puppet-deploy.ini, ~/.puppet-deploy.ini and
+a puppet-deploy.ini in the working directory in that order.
+"""
+import argparse
+import logging
+import logging.config
+import os
+import subprocess
+import sys
+from configparser import ConfigParser
+
+ENV_FOR_GIT = {
+    "PATH": ":".join(
+        [
+            "/opt/puppetlabs/puppet/bin/",
+            "/usr/local/sbin",
+            "/usr/local/bin",
+            "/usr/sbin",
+            "/usr/bin",
+            "/sbin:/bin",
+        ]
+    )
+}
+
+GIT_DIRECTORIES = {}
+
+LOGGER = None
+
+
+def read_ini():
+    global ENV_FOR_GIT, GIT_DIRECTORIES, LOGGER
+    config = ConfigParser()
+    config.read(
+        [
+            "/etc/puppet-deploy.ini",
+            os.path.expanduser("~/.puppet-deploy.ini"),
+            "puppet-deploy.ini",
+        ]
+    )
+    if not config.has_section("puppet-deploy"):
+        logging.error("puppet-deploy section not found in config")
+        sys.exit(1)
+    ENV_FOR_GIT["SSHPASS"] = config.get("puppet-deploy", "ssh_passphrase")
+    branches = config.get("puppet-deploy", "branches").split(",")
+    for branch in branches:
+        GIT_DIRECTORIES[branch] = config.get("branch-directories", branch)
+
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "formatters": {
+                "full": {
+                    "format": "%(asctime)s %(levelname)-8s %(message)s",
+                    "datefmt": "%Y-%m-%d %H:%M:%S",
+                }
+            },
+            "handlers": {
+                "file": {"class": "logging.StreamHandler", "formatter": "full"}
+            },
+            "loggers": {
+                "puppet-deploy": {"handlers": ["file"], "level": "INFO"}
+            },
+        }
+    )
+    LOGGER = logging.getLogger("puppet-deploy")
+
+
+class PuppetDeploy(object):
+
+    def __init__(self, branch_ref):
+        global LOGGER
+        self.log = LOGGER
+        self.branch = branch_ref.split("/")[-1]
+        super().__init__()
+
+    def handle_pull(self) -> None:
+        if self.branch not in GIT_DIRECTORIES:
+            self.log.warn("no directory defined for branch %s", self.branch)
+            return
+        work_dir = GIT_DIRECTORIES[self.branch]
+
+        try:
+            git_proc = subprocess.run(
+                ["sshpass", "-e", "-P", "passphrase", "git", "pull"],
+                env=ENV_FOR_GIT,
+                cwd=work_dir,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=True,
+                universal_newlines=True,
+            )
+            for line in git_proc.stdout.splitlines():
+                self.log.info("git: %s", line)
+        except subprocess.CalledProcessError as e:
+            self.log.error(
+                "could not pull changes for %s: %s", work_dir, e.returncode
+            )
+            for line in e.stdout.splitlines():
+                self.log.info("git: %s", line)
+            self.log.error("error updating the repository in %s", work_dir)
+            sys.exit(1)
+
+        try:
+            r10k_proc = subprocess.run(
+                ["r10k", "puppetfile", "install"],
+                cwd=work_dir,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=True,
+                universal_newlines=True,
+            )
+            for line in r10k_proc.stdout.splitlines():
+                self.log.info("r10k: %s", line)
+        except subprocess.CalledProcessError as e:
+            self.log.error(
+                "could not update modules from Puppetfile: %s", e.returncode
+            )
+            for line in e.stdout.splitlines():
+                self.log.error("r10k: %s", line)
+            self.log.error("error updating modules")
+            sys.exit(1)
+
+        print("updated %s" % work_dir)
+
+
+if __name__ == "__main__":
+    read_ini()
+    parser = argparse.ArgumentParser(prog="puppet-deploy")
+    parser.add_argument("ref")
+    args = parser.parse_args()
+    PuppetDeploy(args.ref).handle_pull()
diff --git a/sitemodules/profiles/files/puppet_server/webhook.service b/sitemodules/profiles/files/puppet_server/webhook.service
new file mode 100644
index 0000000..bdcf291
--- /dev/null
+++ b/sitemodules/profiles/files/puppet_server/webhook.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Small server for creating HTTP endpoints (hooks)
+Documentation=https://github.com/adnanh/webhook/
+ConditionPathExists=/etc/webhook.conf
+
+[Service]
+ExecStart=/usr/bin/webhook -nopanic -port 8000 -hotreload -hooks /etc/webhook.conf
+
+[Install]
+WantedBy=multi-user.target
diff --git a/sitemodules/profiles/manifests/puppet_server.pp b/sitemodules/profiles/manifests/puppet_server.pp
index eb7a4bb..f555193 100644
--- a/sitemodules/profiles/manifests/puppet_server.pp
+++ b/sitemodules/profiles/manifests/puppet_server.pp
@@ -8,10 +8,13 @@
 #
 # @param git_pull_ssh_passphrase passphrase to use for the ssh key to pull
 #                                new code from the control repository
-# @param git_pull_directory      directory where the puppet control repository
-#                                is checked out
-# @param git_pull_tokens         list of tokens that are valid to trigger the
-#                                git pull hook
+# @param git_pull_branches       array of branches to be pulled by the
+#                                puppet-deploy webhook
+# @param git_pull_directories    branch to directory mapping where the puppet
+#                                control repository for a branch is checked
+#                                out
+# @param git_pull_token          token that is valid to trigger the
+#                                puppet-deploy webhook
 #
 # Examples
 # --------
@@ -29,59 +32,85 @@
 # Copyright
 # ---------
 #
-# Copyright 2018 Jan Dittberner
+# Copyright 2018-2020 Jan Dittberner
 class profiles::puppet_server (
   String $git_pull_ssh_passphrase,
-  String $git_pull_directory = '/etc/puppetlabs/code/environments/production',
-  Array[String] $git_pull_tokens,
+  String $git_pull_token,
+  Array[String] $git_pull_branches           = ["master"],
+  Hash[String, String] $git_pull_directories = {
+    'master' => '/etc/puppetlabs/code/environments/production'
+  },
 ) {
-  package { 'sshpass':
-    ensure => installed,
-  }
-
-  package { 'git':
+  package { ['git', 'r10k', 'sshpass', 'webhook']:
     ensure => installed,
   }
 
   file { '/usr/local/sbin/git-pull-hook':
-    ensure  => file,
+    ensure => absent,
+  }
+  file { '/usr/local/sbin/puppet-deploy':
     owner   => 'root',
     group   => 'root',
     mode    => '0750',
-    source  => 'puppet:///modules/profiles/puppet_server/git-pull-hook',
-    require => [Package['sshpass'], Package['git']],
+    source  => 'puppet:///modules/profiles/puppet_server/puppet-deploy',
+    require => [Package['sshpass'], Package['git'], Package['r10k'], Package['webhook']],
   }
 
+  service { 'git-pull-hook':
+    ensure => stopped,
+    enable => false,
+  } ->
   file { '/etc/init.d/git-pull-hook':
+    ensure => absent,
+  }
+  file { '/etc/git-pull-hook.ini':
+    ensure => absent,
+  }
+
+  file { '/etc/puppet-deploy.ini':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0400',
+    content => epp(
+      'profiles/puppet_server/puppet-deploy.ini.epp',
+      {
+        'ssh_passphrase'  => $git_pull_ssh_passphrase,
+        'git_branches'    => $git_pull_branches,
+        'git_directories' => $git_pull_directories,
+      }
+    ),
+  }
+  file { '/etc/systemd/system/webhook.service':
     ensure => file,
     owner  => 'root',
     group  => 'root',
-    mode   => '0755',
-    source => 'puppet:///modules/profiles/puppet_server/git-pull-hook.init.sh'
+    mode   => '0644',
+    source => 'puppet:///modules/profiles/puppet_server/webhook.service',
+  } ~>
+  exec { '/usr/bin/systemctl daemon reload':
+    refreshonly => true,
   }
-
-  file { '/etc/git-pull-hook.ini':
+  file { '/etc/webhook.conf':
     ensure  => file,
     owner   => 'root',
     group   => 'root',
     mode    => '0400',
     content => epp(
-      'profiles/puppet_server/git-pull-hook.ini.epp',
-      {
-        'ssh_passphrase' => $git_pull_ssh_passphrase,
-        'tokens'         => $git_pull_tokens,
-        'git_directory'  => $git_pull_directory,
+      'profiles/puppet_server/webhook.conf.epp', {
+        'token'    => $git_pull_token,
+        'branches' => $git_pull_branches,
       }
-    )
+    ),
   }
 
-  service { 'git-pull-hook':
+  service { 'webhook':
     ensure    => running,
     enable    => true,
-    subscribe => [File['/etc/git-pull-hook.ini'], File['/usr/local/sbin/git-pull-hook']],
+    subscribe => [File['/etc/webhook.conf'], File['/usr/local/sbin/puppet-deploy']],
     require   => [
-      File['/etc/init.d/git-pull-hook'], File['/usr/local/sbin/git-pull-hook'],
-      File['/etc/git-pull-hook.ini'],
+      File['/etc/webhook.conf'], File['/usr/local/sbin/puppet-deploy'],
+      File['/etc/puppet-deploy.ini'], File['/etc/systemd/system/webhook.service'],
     ],
   }
 }
\ No newline at end of file
diff --git a/sitemodules/profiles/templates/puppet_server/git-pull-hook.ini.epp b/sitemodules/profiles/templates/puppet_server/git-pull-hook.ini.epp
deleted file mode 100644
index 2876c16..0000000
--- a/sitemodules/profiles/templates/puppet_server/git-pull-hook.ini.epp
+++ /dev/null
@@ -1,9 +0,0 @@
-<%- | String $ssh_passphrase = undef, String $git_directory = undef, Array[String] $tokens = undef | -%>
-# THIS FILE IS MANAGED BY PUPPET, MANUAL CHANGES WILL BE OVERWRITTEN AT THE
-# NEXT PUPPET RUN.
-
-[git-pull-hook]
-ssh_passphrase=<%= $ssh_passphrase %>
-tokens=<%= $tokens.join(',') %>
-git_directory=<%= $git_directory %>
-logfile=/var/log/git-pull-hook.log
diff --git a/sitemodules/profiles/templates/puppet_server/puppet-deploy.ini.epp b/sitemodules/profiles/templates/puppet_server/puppet-deploy.ini.epp
new file mode 100644
index 0000000..fb23fea
--- /dev/null
+++ b/sitemodules/profiles/templates/puppet_server/puppet-deploy.ini.epp
@@ -0,0 +1,14 @@
+<%- |
+String $ssh_passphrase = undef, Array[String] $git_branches = undef, Hash[String, String] $git_directories = undef
+| -%>
+# THIS FILE IS MANAGED BY PUPPET, MANUAL CHANGES WILL BE OVERWRITTEN AT THE
+# NEXT PUPPET RUN.
+
+[puppet-deploy]
+ssh_passphrase=<%= $ssh_passphrase %>
+branches=<%= $git_branches.join(",") %>
+
+[branch-directories]
+<% $git_directories.each |String $branch, String $directory| { -%>
+<%= $branch %>=<%= $directory %>
+<% } -%>
\ No newline at end of file
diff --git a/sitemodules/profiles/templates/puppet_server/webhook.conf.epp b/sitemodules/profiles/templates/puppet_server/webhook.conf.epp
new file mode 100644
index 0000000..6db1062
--- /dev/null
+++ b/sitemodules/profiles/templates/puppet_server/webhook.conf.epp
@@ -0,0 +1,40 @@
+<%- | String $token = undef, Array[String] $branches = undef | -%>
+[
+  {
+    "id": "puppet-deploy",
+    "execute-command": "/usr/local/sbin/puppet-deploy",
+    "command-working-directory": "/etc/puppetlabs/code/environments",
+    "include-command-output-in-response": true,
+    "include-command-output-in-response-on-error": true,
+    "pass-arguments-to-command": [
+      {
+        "source": "payload",
+        "name": "ref"
+      }
+    ],
+    "trigger-rule": {
+      "and": [
+        {
+          "match": {
+            "type": "value",
+            "value": "<%= $token %>",
+            "parameter": {
+              "source": "header",
+              "name": "Authentication"
+            }
+          }
+        },
+        {
+          "match": {
+            "type": "regex",
+            "regex": "^refs/heads/(<%= $branches.join("|") %>)$",
+            "parameter": {
+              "source": "payload",
+              "name": "ref"
+            }
+          }
+        }
+      ]
+    }
+  }
+]
\ No newline at end of file