Setup automatic updates of icinga2/conf.d from git

* add git hook for icinga2 on monitor

7 files changed, 377 insertions, 11 deletions

diff --git a/hieradata/nodes/monitor.yaml b/hieradata/nodes/monitor.yaml
index 17cf595..de83020 100644
--- a/hieradata/nodes/monitor.yaml
+++ b/hieradata/nodes/monitor.yaml
@@ -7,6 +7,19 @@ profiles::base::admins:
 profiles::base::crl_job_enable: true
 profiles::base::crl_job_services:
   - apache2
+profiles::icinga2_master::git_pull_ssh_passphrase: >
+    ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEw
+    DQYJKoZIhvcNAQEBBQAEggEADAUF/OAtThNdlPwEwrPKAVwl+wTJbirFEWxL
+    rJzE1qe+NSncOqD+G6KNOBQRRXfv/sf81+AnTCahM1/kv5TPILrUgXoxW5c0
+    IXC6OlDfaIab8kcC45wn2yj/igZnW1Xvix3n268pEfRnNDjUSFwrgbmaLtoV
+    ovDLZvQOlWntN8VUuYaDr66XRSEy4AGcmCMUms+6RQqdupWfOCrHtnTtVyyN
+    enQUKr0+ndlnzIkXiU4ghOjExFzGJ8BxGyKTMeQ72k2GZlDPUk72sixZ647k
+    f7CbzXToutyFqieOdNtkAKDY2T3ij03Wd3JhNWTu1Jbe4G/AQgsxiTwETdqh
+    /QgjjDBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBBKOeRdLS8fFyoc08hO
+    BnsVgDBWDpuwBbC31j4g02xKE0tbvazTE8zhkH6iS5mIrL3R5heLvDwquYia
+    pUh+MxqObAs=]
+profiles::icinga2_master::git_pull_tokens:
+  - ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAkVwRv0eW4NTYjfoKx2MVU9WeElhZQIc6CPUnPv4NEM2mUebo5pZg3cvf0fejPw55E33H8QELhMjaBvuOjbgPeA6uCxPMCBADkN6F+V4PRDDgUqtjr1tcA5U05ZEe2oOjOoVI0H2AjLZGevymxypdOCd582vKAApJDox2Hfl2aSuDYLslHYUyIqnECutQR7VgZuv84C/MmDY9J6/xsxesuIKEGRYvgW0DrYqCi4+SBNcFs3k/u2fnP+cXBnCzOp/CvYOwNl9Wkfolj0Ucbh7Afc2ian+ciH2vKODKXck5eUcBx+VrYFXyEJ45Hp/+taYluWClOoq4O6QH+P7yE3Jo2TBcBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCAlTDcDFyyZASZwDyijejegDA6wDKAij0JTyXb0+jxTATAD+Sxpp/BWbmJSvDuC5MpwziJfJsH1tvoM0JOPYapRkc=]
 profiles::icinga2_master::web2_database_password: >
     ENC[PKCS7,MIIBmQYJKoZIhvcNAQcDoIIBijCCAYYCAQAxggEhMIIBHQIBADAFMAACAQEw
     DQYJKoZIhvcNAQEBBQAEggEAIgd5qF6rnFWYhyo38MRacrz2VcYdoni/m8Zd
diff --git a/sitemodules/profiles/files/icinga2_master/icinga2-git-pull-hook b/sitemodules/profiles/files/icinga2_master/icinga2-git-pull-hook
new file mode 100644
index 0000000..a0d3711
--- /dev/null
+++ b/sitemodules/profiles/files/icinga2_master/icinga2-git-pull-hook
@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+
+"""
+This script takes care of updating the configuration in a directory by
+performing git pull when triggered via an HTTP request to port 8000.
+
+The script needs the sshpass and git packages installed.
+
+Configuration is read from /etc/default/icinga2-git-pull-hook.ini,
+~/.icinga2-git-pull-hook.ini and a icinga2-git-pull-hook.ini in the working
+directory in that order.
+"""
+
+import logging
+import logging.config
+import os
+import subprocess
+from configparser import ConfigParser
+from http import HTTPStatus
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+ENV_FOR_GIT = {"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}
+
+TOKENS = []
+GIT_DIRECTORY = ""
+GIT_REPOSITORY = ""
+GIT_BRANCH = ""
+
+LOGGER = None
+
+
+def read_ini():
+    global ENV_FOR_GIT, TOKENS, GIT_DIRECTORY, GIT_REPOSITORY, GIT_BRANCH, LOGGER
+    config = ConfigParser()
+    config.read(
+        [
+            "/etc/default/icinga2-git-pull-hook.ini",
+            os.path.expanduser("~/.icinga2-git-pull-hook.ini"),
+            "icinga2-git-pull-hook.ini",
+        ]
+    )
+    ENV_FOR_GIT["SSHPASS"] = config["icinga2-git-pull-hook"]["ssh_passphrase"]
+    TOKENS = [
+        token.strip() for token in config["icinga2-git-pull-hook"]["tokens"].split(",")
+    ]
+    GIT_DIRECTORY = config["icinga2-git-pull-hook"]["git_directory"]
+    GIT_REPOSITORY = config["icinga2-git-pull-hook"]["git_repository"]
+    GIT_BRANCH = config["icinga2-git-pull-hook"]["git_branch"]
+
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "formatters": {
+                "full": {
+                    "format": "%(asctime)s %(levelname)-8s %(message)s",
+                    "datefmt": "%Y-%m-%d %H:%M:%S",
+                }
+            },
+            "handlers": {
+                "file": {
+                    "class": "logging.FileHandler",
+                    "filename": config.get("icinga2-git-pull-hook", "logfile"),
+                    "formatter": "full",
+                }
+            },
+            "loggers": {
+                "icinga2-git-pull-hook": {"handlers": ["file"], "level": "INFO"}
+            },
+        }
+    )
+    LOGGER = logging.getLogger("icinga2-git-pull-hook")
+
+
+class GitHookRequestHandler(BaseHTTPRequestHandler):
+    """
+    Custom HTTP request handler for updating a git repository when called
+    with a known authentication token in an "Authentication" HTTP header.
+    """
+
+    def __init__(self, request, client_address, server):
+        global LOGGER
+        self.log = LOGGER
+        super().__init__(request, client_address, server)
+
+    def _send_data(self, message):
+        self.send_header("Content-Type", "text/plain; charset=utf8")
+        self.end_headers()
+        self.wfile.write(("%s\r\n" % message).encode("UTF-8"))
+
+    def _handle_pull(self):
+        try:
+            git_proc = subprocess.run(
+                [
+                    "sshpass",
+                    "-e",
+                    "-P",
+                    "passphrase",
+                    "git",
+                    "subtree",
+                    "pull",
+                    "--prefix",
+                    "icinga2/conf.d",
+                    GIT_REPOSITORY,
+                    GIT_BRANCH,
+                ],
+                env=ENV_FOR_GIT,
+                cwd="/etc",
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=True,
+                universal_newlines=True,
+            )
+            for line in git_proc.stdout.splitlines():
+                self.log_message("git: %s", line)
+        except subprocess.CalledProcessError as e:
+            self.log_error(
+                "Could not pull changes for %s: %s", GIT_DIRECTORY, e.returncode
+            )
+            for line in e.stdout.splitlines():
+                self.log_message("git: %s", line)
+            self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR)
+            self._send_data("Error updating the repository.")
+            return
+        self._send_data("updated %s" % GIT_DIRECTORY)
+        try:
+            icinga2_config_check_proc = subprocess.run(
+                ["/usr/sbin/icinga2", "daemon", "-C"],
+                cwd=GIT_DIRECTORY,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=True,
+                universal_newlines=True,
+            )
+            for line in icinga2_config_check_proc.stdout.splitlines():
+                self.log_message("icinga2: %s", line)
+        except subprocess.CalledProcessError as e:
+            self.log_error("configuration check failed: %d", e.returncode)
+            for line in e.stdout.splitlines():
+                self.log_message("icinga2: %s", line)
+            self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR)
+            self._send_data("Error updating configuration.")
+            return
+        try:
+            icinga2_reload_check_proc = subprocess.run(
+                ["/bin/systemctl", "reload", "icinga2.service"],
+                cwd=GIT_DIRECTORY,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=True,
+                universal_newlines=True,
+            )
+            for line in icinga2_reload_check_proc.stdout.splitlines():
+                self.log_message("systemctl: %s", line)
+        except subprocess.CalledProcessError as e:
+            self.log_error("reload failed: %s", e.returncode)
+            for line in e.stdout.splitlines():
+                self.log_message("systemctl: %s", line)
+            self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR)
+            self._send_data("Error reloading icinga2.")
+            return
+
+        self.send_response(HTTPStatus.OK)
+        self._send_data("Updated icinga2 configuration")
+        return
+
+    # noinspection PyPep8Naming
+    def do_GET(self):
+        """
+        Handle GET requests, requests to /health are allowed for every caller.
+        """
+        if self.path == "/health":
+            self.send_response(HTTPStatus.OK)
+            self._send_data("I'm healthy!")
+        else:
+            self.send_error(
+                HTTPStatus.NOT_FOUND, "You requested something I do not understand."
+            )
+
+    # noinspection PyPep8Naming
+    def do_POST(self):
+        """
+        Handle POST requests requests to / need a valid token in the
+        "Authentication" HTTP header and trigger a git pull in the configured
+        directory.
+        """
+        if self.path == "/":
+            if self.headers["Authentication"] in [token for token in TOKENS]:
+                self._handle_pull()
+            else:
+                self.send_error(
+                    HTTPStatus.UNAUTHORIZED,
+                    'You have to send a valid token in the "Authentication" header.',
+                )
+        else:
+            self.send_error(
+                HTTPStatus.NOT_FOUND, "You requested something I do not understand."
+            )
+
+    def log_error(self, format, *args):
+        self.log.error("%s - %s" % (self.address_string(), format), *args)
+
+    def log_message(self, format, *args):
+        self.log.info("%s - %s" % (self.address_string(), format), *args)
+
+
+def run(server_class=HTTPServer, handler_class=GitHookRequestHandler):
+    server_address = ("", 8000)
+    httpd = server_class(server_address, handler_class)
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    read_ini()
+    run()
diff --git a/sitemodules/profiles/files/icinga2_master/icinga2-git-pull-hook.service b/sitemodules/profiles/files/icinga2_master/icinga2-git-pull-hook.service
new file mode 100644
index 0000000..a423a5f
--- /dev/null
+++ b/sitemodules/profiles/files/icinga2_master/icinga2-git-pull-hook.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Icinga2 configuration git pull hook
+Requires=icinga2.service
+
+[Service]
+ExecStart=/usr/local/sbin/icinga2-git-pull-hook
+WorkingDirectory=/etc/icinga2/conf.d
+
+[Install]
+WantedBy=multi-user.target
diff --git a/sitemodules/profiles/manifests/debarchive.pp b/sitemodules/profiles/manifests/debarchive.pp
index fd79ed6..82888b5 100644
--- a/sitemodules/profiles/manifests/debarchive.pp
+++ b/sitemodules/profiles/manifests/debarchive.pp
@@ -57,6 +57,7 @@ class profiles::debarchive (
 
   include profiles::base
   include profiles::apache_common
+  include profiles::systemd_reload
 
   package{ ['rssh', 'reprepro', 'inoticoming']:
     ensure => latest,
@@ -295,10 +296,7 @@ class profiles::debarchive (
       File[$trusted_keyring],
       User['debarchive'],
     ],
-  } ~>
-  exec { 'reload systemd configuration after changes to service file':
-    command     => '/bin/systemctl daemon-reload',
-    refreshonly => true,
+    notify  => Exec['reload systemd configuration'],
   }
 
   service { 'debarchive-inoticoming':
diff --git a/sitemodules/profiles/manifests/icinga2_master.pp b/sitemodules/profiles/manifests/icinga2_master.pp
index acdaab8..258345c 100644
--- a/sitemodules/profiles/manifests/icinga2_master.pp
+++ b/sitemodules/profiles/manifests/icinga2_master.pp
@@ -7,13 +7,32 @@
 # Parameters
 # ----------
 #
-# @param ido_database_password  database password for Icinga2 IDO database
-# @param web2_database_password database password for IcingaWeb2 database
-# @param api_users         Icinga2 API users
-# @param pki_ticket_salt   Ticket salt for API endpoint
-# @param ca_key            Icinga2 CA private key content
-# @param ca_certificate    Icinga2 CA certificate content
-# @param $icingaweb_admins List of icingaweb admin users
+# @param ido_database_password   database password for Icinga2 IDO database
+#
+# @param web2_database_password  database password for IcingaWeb2 database
+#
+# @param api_users               Icinga2 API users
+#
+# @param pki_ticket_salt         Ticket salt for API endpoint
+#
+# @param ca_key                  Icinga2 CA private key content
+#
+# @param ca_certificate          Icinga2 CA certificate content
+#
+# @param $icingaweb_admins       List of icingaweb admin users
+#
+# @param git_pull_ssh_passphrase passphrase to use for the ssh key to pull new
+#                                configuration from the configuration repository
+#
+# @param git_pull_directory      directory where the icinga2 configuration
+#                                is checked out
+#
+# @param git_pull_tokens         list of tokens that are valid to trigger the
+#                                git pull hook
+#
+# @param git_repository          configuration git repository
+#
+# @param git_branch              configuration branch in the git repository
 #
 # Examples
 # --------
@@ -40,8 +59,14 @@ class profiles::icinga2_master (
   String $ca_key,
   String $ca_certificate,
   Array[String] $icingaweb_admins = ['icingaadmin'],
+  String $git_pull_ssh_passphrase,
+  String $git_pull_directory = '/etc/icinga2/conf.d',
+  Array[String] $git_pull_tokens,
+  String $git_repository = 'icinga2git@git:/var/lib/git/cacert-icinga2-conf_d.git',
+  String $git_branch = 'master',
 ) {
   include profiles::icinga2_common
+  include profiles::systemd_reload
   include postgresql::server
 
   class { '::icinga2':
@@ -140,4 +165,59 @@ class profiles::icinga2_master (
     permissions => '*',
     require     => Class['::icingaweb2'],
   }
+
+  package { ['sshpass', 'git']:
+    ensure => installed,
+  }
+
+  $git_pull_hook = '/usr/local/sbin/icinga2-git-pull-hook'
+  $git_pull_hook_config = '/etc/default/icinga2-git-pull-hook.ini'
+  $git_pull_hook_service = '/etc/systemd/system/icinga2-git-pull-hook.service'
+
+  file { $git_pull_hook:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0750',
+    source  => 'puppet:///modules/profiles/icinga2_master/icinga2-git-pull-hook',
+    require => [Package['sshpass'], Package['git']],
+    notify => Exec['reload systemd configuration'],
+  }
+
+  file { $git_pull_hook_service:
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/profiles/icinga2_master/icinga2-git-pull-hook.service',
+    notify => Exec['reload systemd configuration'],
+  }
+
+  file { $git_pull_hook_config:
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0400',
+    content => epp(
+      'profiles/icinga2_master/icinga2-git-pull-hook.ini.epp',
+      {
+        'ssh_passphrase' => $git_pull_ssh_passphrase,
+        'tokens'         => $git_pull_tokens,
+        'git_directory'  => $git_pull_directory,
+        'git_repository' => $git_repository,
+        'git_branch'     => $git_branch,
+      }
+    ),
+    notify => Exec['reload systemd configuration'],
+  }
+
+  service { 'icinga2-git-pull-hook':
+    ensure  => running,
+    enable  => true,
+    require => [
+      File[$git_pull_hook],
+      File[$git_pull_hook_config],
+      File[$git_pull_hook_service],
+    ],
+  }
 }
diff --git a/sitemodules/profiles/manifests/systemd_reload.pp b/sitemodules/profiles/manifests/systemd_reload.pp
new file mode 100644
index 0000000..89e76ad
--- /dev/null
+++ b/sitemodules/profiles/manifests/systemd_reload.pp
@@ -0,0 +1,35 @@
+# Class: profiles::systemd_reload
+# ===============================
+#
+# systemd daemon reload execution that can be triggerd from other resources by
+# notifying Exec['reload systemd configuration'].
+#
+# This manifest is meant to be included from other manifests.
+#
+# Examples
+# --------
+#
+# @example
+#   include profiles::systemd_reload
+#
+#   file { 'myfile':
+#     source => 'some_source',
+#     notify => Exec['reload systemd configuration'],
+#   }
+#
+# Authors
+# -------
+#
+# Jan Dittberner <jandd@cacert.org>
+#
+# Copyright
+# ---------
+#
+# Copyright 2019 Jan Dittberner
+class profiles::systemd_reload (
+) {
+  exec { 'reload systemd configuration':
+    command     => '/bin/systemctl daemon-reload',
+    refreshonly => true,
+  }
+}
diff --git a/sitemodules/profiles/templates/icinga2_master/icinga2-git-pull-hook.ini.epp b/sitemodules/profiles/templates/icinga2_master/icinga2-git-pull-hook.ini.epp
new file mode 100644
index 0000000..02d5b0c
--- /dev/null
+++ b/sitemodules/profiles/templates/icinga2_master/icinga2-git-pull-hook.ini.epp
@@ -0,0 +1,16 @@
+<%- | String $ssh_passphrase,
+      String $git_directory,
+      String $git_repository,
+      String $git_branch,
+      Array[String] $tokens
+| -%>
+# THIS FILE IS MANAGED BY PUPPET, MANUAL CHANGES WILL BE OVERWRITTEN AT THE
+# NEXT PUPPET RUN.
+
+[icinga2-git-pull-hook]
+ssh_passphrase=<%= $ssh_passphrase %>
+tokens=<%= $tokens.join(',') %>
+git_directory=<%= $git_directory %>
+logfile=/var/log/git-pull-hook.log
+git_repository=<%= $git_repository %>
+git_branch=<%= $git_branch %>