[nomad] Add nomad and consul playbooks

ansible/hosts

@@ -16,10 +16,21 @@ weewx.local
 books.local
 photos.local
 podcasts.local
-fifteen5.local
 grafana.local
 test.local
 jellyfin.local
+nomad00.local
+nomad01.local
+nomad02.local
+nomad03.local
+nomad10.local
+nomad11.local
+nomad12.local
+nomad13.local
+nomad-agent0.local
+nomad-agent1.local
+nomad-agent2.local
+nomad-agent3.local
 
 [bhyve]
 mopidy.local ansible_become=true
@@ -33,6 +44,8 @@ prometheus.local ansible_become=true
 dione.local ansible_become=true
 paaliaq.local ansible_become=true
 kiviuq.local ansible_become=true
+kari.local ansible_become=true
+atlas.local ansible_become=true
 
 [hosted]
 box.unbl.ink
@@ -41,6 +54,7 @@ pandora.unbl.ink
 
 [ubuntu]
 mopidy.local ansible_become=true
+kari.local ansible_become=true
 box.unbl.ink
 box.castine.town
 pandora.unbl.ink
@@ -48,6 +62,7 @@ pandora.unbl.ink
 [freebsd]
 paaliaq.local ansible_become=true
 kiviuq.local ansible_become=true
+atlas.local ansible_become=true
 bastion.local ansible_become=true
 search.local
 db.local
@@ -65,8 +80,19 @@ weewx.local
 books.local
 photos.local
 podcasts.local
-fifteen5.local
 jellyfin.local
+nomad00.local
+nomad01.local
+nomad02.local
+nomad03.local
+nomad10.local
+nomad11.local
+nomad12.local
+nomad13.local
+nomad-agent0.local
+nomad-agent1.local
+nomad-agent2.local
+nomad-agent3.local
 
 [mailinabox]
 box.unbl.ink
@@ -91,7 +117,6 @@ unbound.local
 [webapps]
 dev-pbp.local
 dev-ff.local
-fifteen5.local
 
 [gogs]
 gogs.local
@@ -121,7 +146,7 @@ tor.local
 pbp.local
 
 [five]
-fifteen5.local
+dev-ff.local
 
 [db]
 db.local
@@ -156,3 +181,23 @@ redis.local
 
 [test]
 test.local
+
+[consul-nodes-dc1]
+nomad00.local
+nomad01.local
+nomad02.local
+nomad03.local
+
+[consul-nodes-dc2]
+nomad10.local
+nomad11.local
+nomad12.local
+nomad13.local
+
+[nomad-agents-dc1]
+nomad-agent0.local
+nomad-agent1.local
+
+[nomad-agents-dc2]
+nomad-agent2.local
+nomad-agent3.local

ansible/playbook.yml

@@ -97,6 +97,28 @@
   roles:
     - role: jellyfin
 
+- hosts: consul-nodes-dc1
+  roles:
+    - role: dnsmasq
+    - role: consul
+    - role: nomad
+
+- hosts: consul-nodes-dc2
+  roles:
+    - role: dnsmasq
+    - role: consul
+    - role: nomad
+
+- hosts: nomad-agents-dc1
+  roles:
+    - role: dnsmasq
+    - role: nomad-agent
+
+- hosts: nomad-agents-dc2
+  roles:
+    - role: dnsmasq
+    - role: nomad-agent
+
 - hosts: test
   roles:
     - role: jails

ansible/roles/consul/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Consul restarted
+  service:
+    name: consul
+    state: restarted

ansible/roles/consul/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: Consul installed
+  community.general.pkgng:
+    name: "consul"
+    state: latest
+  tags:
+    - dc1
+    - dc2
+
+- name: Consul enabled
+  shell: sysrc consul_enable="YES"
+  tags:
+    - dc1
+    - dc2
+  notify: Consul restarted
+
+- name: Consul syslog output
+  shell: sysrc consul_syslog_output_enable="YES"
+  tags:
+    - dc1
+    - dc2
+  notify: Consul restarted
+
+- name: Consul dc1 nodes configured
+  shell: sysrc consul_args="-server -client 0.0.0.0 -ui -bootstrap-expect=3 -datacenter dc1 -retry-join-wan nomad11.local -retry-join nomad00.local -retry-join nomad01.local -retry-join nomad02.local -retry-join nomad03.local"
+  tags:
+    - dc1
+  notify: Consul restarted
+
+- name: Consul dc2 nodes configured
+  shell: sysrc consul_args="-server -client 0.0.0.0 -ui -bootstrap-expect=3 -datacenter dc2 -retry-join-wan nomad01.local -retry-join nomad10.local -retry-join nomad11.local -retry-join nomad12.local -retry-join nomad13.local"
+  tags:
+    - dc2
+  notify: Consul restarted
+
+- name: Consul dc1 agent configured
+  shell: sysrc consul_args="-datacenter dc1 -retry-join consul.service.dc1.consul"
+  tags:
+    - dc1-agents
+  notify: Consul restarted
+
+- name: Consul dc2 agent configured
+  shell: sysrc consul_args="-datacenter dc2 -retry-join consul.service.dc2.consul"
+  tags:
+    - dc1-agents
+  notify: Consul restarted
+
+- name: Consul started
+  service: name=consul state=started
+  tags:
+    - dc1
+    - dc2

ansible/roles/dnsmasq/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Dnsmasq restarted
+  service:
+    name: dnsmasq
+    state: restarted

ansible/roles/dnsmasq/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Dnsmasq installed
+  community.general.pkgng:
+    name: "dnsmasq"
+    state: latest
+  tags:
+    - dnsmasq
+
+- name: Dnsmasq enabled
+  shell: sysrc dnsmasq_enable="YES"
+  notify: Dnsmasq restarted
+  tags:
+    - dnsmasq
+
+- name: Dnsmasq started
+  service: name=dnsmasq state=started
+  tags:
+    - dnsmasq

ansible/roles/nomad-agent/files/client.hcl

@@ -0,0 +1,22 @@
+data_dir = "/var/lib/nomad/"
+
+client {
+  enabled = true
+  servers = ["nomad00.local:4647","nomad01.local:4647","nomad02.local:4647","nomad03.local:4647"]
+}
+
+plugin "docker" {
+  config {
+    gc {
+      dangling_containers {
+        enabled = false
+      }
+    }
+  }
+}
+
+plugin "raw_exec" {
+  config {
+    enabled = true
+  }
+}

ansible/roles/nomad-agent/files/http-echo

@@ -0,0 +1,127 @@
+#!/usr/local/bin/python3.8
+"""
+A Simple Python HTTP server that echos the request in the response.
+"""
+
+import socket
+import argparse
+from six.moves.urllib import parse
+import email.message
+
+try:
+    from email.generator import BytesGenerator
+except ImportError:
+    # BBB Python 2 compatibility
+    from email.generator import Generator as BytesGenerator
+
+from six.moves import BaseHTTPServer
+
+parser = argparse.ArgumentParser(
+    description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter
+)
+parser.add_argument(
+    "--address",
+    "-a",
+    default="0.0.0.0",
+    help="Hostname or IP address to accept requests on.",
+)
+parser.add_argument(
+    "--port",
+    "-p",
+    help="Port to accept requests on.  "
+    "If not specified, use the first available port after 8000.",
+)
+
+
+class EchoHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    """
+    A Simple Python HTTP server that echos the request in the response.
+    """
+
+    def do_GET(self):
+        """
+        Echo a request without a body.
+        """
+        message = self.get_message()
+        self.send_head()
+        BytesGenerator(self.wfile).flatten(message, unixfrom=False)
+
+    do_HEAD = do_GET
+    do_OPTIONS = do_GET
+    do_DELETE = do_GET
+
+    def do_POST(self):
+        """
+        Echo a request with a body.
+        """
+        message = self.get_message()
+        message.set_payload(self.rfile.read(int(self.headers["Content-Length"])))
+        self.send_head()
+        BytesGenerator(self.wfile).flatten(message, unixfrom=False)
+
+    do_PUT = do_POST
+    do_PATCH = do_POST
+
+    def send_head(self):
+        """
+        Send all the basic, required headers.
+        """
+        self.send_response(200)
+        self.send_header("Content-Type", "text/rfc822-headers; charset=UTF-8")
+        self.send_header("Last-Modified", self.date_time_string())
+        self.end_headers()
+
+    def get_message(self):
+        """
+        Assemble the basic message including query parameters.
+        """
+        message = email.message.Message()
+        message["Method"] = self.command
+        message["Path"] = self.path
+
+        server_url = parse.SplitResult(
+            "http",
+            "{0}:{1}".format(self.server.server_name, self.server.server_port),
+            "",
+            "",
+            "",
+        )
+        request_url = parse.urlsplit(server_url.geturl() + self.path)
+        for header, value in parse.parse_qs(request_url.query).items():
+            message.add_header(header, value[0])
+
+        return message
+
+
+def main(args=None, default_port=8000):
+    """
+    Run the echo HTTP server.
+    """
+    args = parser.parse_args(args)
+
+    port = args.port
+    if port is None:
+        port = default_port
+        bound = False
+        while not bound:
+            try:
+                httpd = BaseHTTPServer.HTTPServer(
+                    (args.address, port), EchoHTTPRequestHandler
+                )
+            except socket.error:
+                port += 1
+                if port > 65535:
+                    raise ValueError("No available port found")
+            else:
+                bound = True
+    else:
+        httpd = BaseHTTPServer.HTTPServer(
+            (args.address, int(port)), EchoHTTPRequestHandler
+        )
+
+    print("Echoing HTTP at http://{0}:{1} ...".format(args.address, port))
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/nomad-agent/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Nomad restarted
+  service:
+    name: nomad
+    state: restarted

ansible/roles/nomad-agent/tasks/main.yml

@@ -0,0 +1,88 @@
+---
+- name: Nomad installed
+  community.general.pkgng:
+    name: "nomad,py38-six"
+    state: latest
+  tags:
+    - dc1
+    - dc2
+
+- name: Nomad enabled
+  shell: sysrc nomad_enable="YES"
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad debug set
+  shell: sysrc nomad_debug="YES"
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad user configured
+  shell: sysrc nomad_user="root"
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad dc1 configured
+  shell: sysrc nomad_args="-config /usr/local/etc/nomad -network-interface epair0b -dc dc1"
+  tags:
+    - dc1
+  notify: Nomad restarted
+
+- name: Nomad dc2 configured
+  shell: sysrc nomad_args="-config /usr/local/etc/nomad -network-interface epair0b -dc dc2"
+  tags:
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad server.hcl removed
+  file:
+    state: absent
+    dest: /usr/local/etc/nomad/server.hcl
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad client.hcl installed
+  copy:
+    src: client.hcl
+    dest: /usr/local/etc/nomad/client.hcl
+    owner: root
+    mode: 0644
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: http-echo copied
+  copy:
+    src: http-echo
+    dest: /usr/local/bin/http-echo
+    owner: root
+    mode: a+x
+  tags:
+    - dc1
+    - dc2
+
+- name: Nomad dns setup
+  template:
+    src: dnsmasq.conf.j2
+    dest: /usr/local/etc/dnsmasq.conf
+    owner: root
+    mode: 0644
+  tags:
+    - dc1
+    - dc2
+  notify: Dnsmasq restarted
+
+- name: Nomad started
+  service: name=nomad state=started
+  tags:
+    - dc1
+    - dc2

ansible/roles/nomad-agent/templates/dnsmasq.conf.j2

@@ -0,0 +1 @@
+server=/consul/{{hostvars[inventory_hostname]['ansible_default_ipv4']['address']}}#8600

ansible/roles/nomad/files/server.hcl

@@ -0,0 +1,12 @@
+data_dir = "/var/lib/nomad"
+
+advertise {
+  # This should be the IP of THIS MACHINE and must be routable by every node
+  # in your cluster
+  #rpc =
+}
+
+server {
+  enabled          = true
+  bootstrap_expect = 3
+}

ansible/roles/nomad/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Nomad restarted
+  service:
+    name: nomad
+    state: restarted

ansible/roles/nomad/tasks/main.yml

@@ -0,0 +1,80 @@
+---
+- name: Nomad installed
+  community.general.pkgng:
+    name: "nomad"
+    state: latest
+  tags:
+    - dc1
+    - dc2
+
+- name: Nomad enabled
+  shell: sysrc nomad_enable="YES"
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad debug set
+  shell: sysrc nomad_debug="YES"
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad dc1 configured
+  shell: sysrc nomad_args="-config /usr/local/etc/nomad -network-interface epair0b -dc dc1"
+  tags:
+    - dc1
+  notify: Nomad restarted
+
+- name: Nomad dc2 configured
+  shell: sysrc nomad_args="-config /usr/local/etc/nomad -network-interface epair0b -dc dc2"
+  tags:
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad server.hcl installed
+  copy:
+    src: server.hcl
+    dest: /usr/local/etc/nomad/server.hcl
+    owner: root
+    mode: 0644
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad client1.hcl removed
+  file:
+    state: absent
+    dest: /usr/local/etc/nomad/client1.hcl
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad client2.hcl removed
+  file:
+    state: absent
+    dest: /usr/local/etc/nomad/client2.hcl
+  tags:
+    - dc1
+    - dc2
+  notify: Nomad restarted
+
+- name: Nomad dns setup
+  template:
+    src: dnsmasq.conf.j2
+    dest: /usr/local/etc/dnsmasq.conf
+    owner: root
+    mode: 0644
+  tags:
+    - dc1
+    - dc2
+  notify: Dnsmasq restarted
+
+- name: Nomad started
+  service: name=nomad state=started
+  tags:
+    - dc1
+    - dc2

ansible/roles/nomad/templates/dnsmasq.conf.j2

@@ -0,0 +1 @@
+server=/consul/{{hostvars[inventory_hostname]['ansible_default_ipv4']['address']}}#8600