added support for Särimner
This commit is contained in:
Erik Bergström
parent
b35c7a07bc
commit
6a1eab34c0
15 changed files with 1916 additions and 0 deletions
|
|
@ -1,2 +1,3 @@
|
|||
tag="eid-ops"
|
||||
addhost_ssh_args="-o ProxyCommand 'ssh root@jmp.komreg.net -W %h:%p'"
|
||||
repo=git://git.nordu.net/eid-ops
|
||||
|
|
|
|||
48
fe-common/overlay/etc/hiera/data/group.yaml
Normal file
48
fe-common/overlay/etc/hiera/data/group.yaml
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
---
|
||||
sunet_frontend:
|
||||
|
||||
load_balancer:
|
||||
api_imagetag: 'staging'
|
||||
exabgp_imagetag: 'staging'
|
||||
port80_acme_c_backend: 'letsencrypt_acme-c.sunet.se'
|
||||
static_backends:
|
||||
letsencrypt_acme-c.sunet.se.se:
|
||||
- '89.45.232.90' # note: port 80
|
||||
|
||||
peers:
|
||||
loke.sunet.se:
|
||||
as: '65434'
|
||||
remote_ip: '130.242.124.130'
|
||||
loke.sunet.se_v6:
|
||||
as: '65434'
|
||||
remote_ip: '2001:6b0:7:127::2'
|
||||
se-tug-rs-1.sunet.se:
|
||||
as: '65434'
|
||||
remote_ip: '130.242.125.111'
|
||||
se-tug-rs-1.sunet.se_v6:
|
||||
as: '65434'
|
||||
remote_ip: '2001:6b0:8:4::111'
|
||||
|
||||
|
||||
websites2_disabled:
|
||||
|
||||
'www':
|
||||
site_name: 'www.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.61', '2001:6b0:54:fe::61']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.62', '2001:6b0:54:fe::62']
|
||||
backends:
|
||||
default:
|
||||
'www-fre-1.eduid.se':
|
||||
ips: ['130.242.130.200']
|
||||
server_args: 'ssl check verify none'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
varnish_enabled: true
|
||||
varnish_imagetag: 'staging'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
197
fe-common/overlay/etc/hiera/data/group.yaml~
Normal file
197
fe-common/overlay/etc/hiera/data/group.yaml~
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
---
|
||||
sunet_frontend:
|
||||
|
||||
load_balancer:
|
||||
api_imagetag: 'staging'
|
||||
exabgp_imagetag: 'staging'
|
||||
port80_acme_c_backend: 'letsencrypt_acme-c.dev.eduid.se'
|
||||
static_backends:
|
||||
letsencrypt_acme-c.dev.eduid.se:
|
||||
- '2001:6b0:54:c3::acac:80' # note: port 80
|
||||
|
||||
peers:
|
||||
loke.sunet.se:
|
||||
as: '65434'
|
||||
remote_ip: '130.242.124.130'
|
||||
loke.sunet.se_v6:
|
||||
as: '65434'
|
||||
remote_ip: '2001:6b0:7:127::2'
|
||||
se-tug-rs-1.sunet.se:
|
||||
as: '65434'
|
||||
remote_ip: '130.242.125.111'
|
||||
se-tug-rs-1.sunet.se_v6:
|
||||
as: '65434'
|
||||
remote_ip: '2001:6b0:8:4::111'
|
||||
|
||||
|
||||
websites2:
|
||||
|
||||
'www':
|
||||
site_name: 'www.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.61', '2001:6b0:54:fe::61']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.62', '2001:6b0:54:fe::62']
|
||||
backends:
|
||||
default:
|
||||
'www-fre-1.eduid.se':
|
||||
ips: ['130.242.130.200']
|
||||
server_args: 'ssl check verify none'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
varnish_enabled: true
|
||||
varnish_imagetag: 'staging'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
'idp':
|
||||
site_name: 'idp.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.21', '2001:6b0:54:fe::21']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.22', '2001:6b0:54:fe::22']
|
||||
backends:
|
||||
default:
|
||||
'idp-tug-1.eduid.se':
|
||||
ips: ['130.242.131.18']
|
||||
server_args: 'ssl check verify none cookie tug-1'
|
||||
'idp-fre-1.eduid.se':
|
||||
ips: ['130.242.130.214']
|
||||
server_args: 'ssl check verify none cookie fre-1'
|
||||
'mfa_add':
|
||||
'kvm-tug-1.eduid.se':
|
||||
ips: ['130.242.130.217']
|
||||
server_args: 'check'
|
||||
csp_app_src: 'https://idp.dev.eduid.se'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
'dashboard':
|
||||
site_name: 'dashboard.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.51', '2001:6b0:54:fe::51']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.52', '2001:6b0:54:fe::52']
|
||||
backends:
|
||||
default:
|
||||
'dash-tug-1.eduid.se':
|
||||
ips: ['130.242.130.197']
|
||||
server_args: 'ssl check verify none check cookie tug-1 weight 25'
|
||||
'dash-fre-1.eduid.se':
|
||||
ips: ['130.242.130.213']
|
||||
server_args: 'ssl check verify none check cookie fre-1 weight 25'
|
||||
'apps-tug-1.eduid.se':
|
||||
ips: ['130.242.130.206']
|
||||
server_args: 'ssl check verify none check cookie apps-tug-1 weight 25'
|
||||
'apps-fre-1.eduid.se':
|
||||
ips: ['130.242.130.207']
|
||||
server_args: 'ssl check verify none check cookie apps-fre-1 weight 25'
|
||||
old:
|
||||
'dash-tug-1.eduid.se':
|
||||
ips: ['130.242.130.197']
|
||||
server_args: 'ssl check verify none check cookie tug-1'
|
||||
'dash-fre-1.eduid.se':
|
||||
ips: ['130.242.130.213']
|
||||
server_args: 'ssl check verify none check cookie fre-1'
|
||||
new:
|
||||
'apps-tug-1.eduid.se':
|
||||
ips: ['130.242.130.206']
|
||||
server_args: 'ssl check verify none check cookie apps-tug-1'
|
||||
'apps-fre-1.eduid.se':
|
||||
ips: ['130.242.130.207']
|
||||
server_args: 'ssl check verify none check cookie apps-fre-1'
|
||||
csp_app_src: 'https://dashboard.dev.eduid.se'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
'signup':
|
||||
site_name: 'signup.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.41', '2001:6b0:54:fe::41']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.42', '2001:6b0:54:fe::42']
|
||||
backends:
|
||||
default:
|
||||
'signup-tug-1.eduid.se':
|
||||
ips: ['130.242.130.212']
|
||||
server_args: 'ssl check verify none cookie tug-1'
|
||||
csp_app_src: 'https://signup.dev.eduid.se'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
csp_google_src: 'https://www.google.com/recaptcha/'
|
||||
csp_gstatic_src: 'https://www.gstatic.com/recaptcha/'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
'support':
|
||||
site_name: 'support.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.31', '2001:6b0:54:fe::31']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.32', '2001:6b0:54:fe::32']
|
||||
backends:
|
||||
default:
|
||||
'dash-tug-1.eduid.se':
|
||||
ips: ['130.242.130.197']
|
||||
server_args: 'ssl check verify none cookie tug-1'
|
||||
'dash-fre-1.eduid.se':
|
||||
ips: ['130.242.130.213']
|
||||
server_args: 'ssl check verify none cookie fre-1'
|
||||
csp_app_src: 'https://support.dev.eduid.se'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
'selegop':
|
||||
site_name: 'op1.se-leg.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.36', '2001:6b0:54:fe::36']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.37', '2001:6b0:54:fe::37']
|
||||
backends:
|
||||
default:
|
||||
'dash-tug-1.eduid.se':
|
||||
ips: ['130.242.130.197']
|
||||
server_args: 'check verify none cookie tug-1'
|
||||
'dash-fre-1.eduid.se':
|
||||
ips: ['130.242.130.213']
|
||||
server_args: 'check verify none cookie fre-1'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
haproxy_imagetag: 'staging'
|
||||
|
||||
'monitor':
|
||||
site_name: 'monitor.dev.eduid.se'
|
||||
frontends:
|
||||
'fe-fre-1.eduid.se':
|
||||
ips: ['130.242.131.11', '2001:6b0:54:fe::11']
|
||||
'fe-tug-1.eduid.se':
|
||||
ips: ['130.242.131.12', '2001:6b0:54:fe::12']
|
||||
backends:
|
||||
default:
|
||||
'monitor-fre-1.eduid.se':
|
||||
ips: ['130.242.130.218']
|
||||
server_args: 'ssl check verify none'
|
||||
csp_app_src: 'https://monitor.dev.eduid.se'
|
||||
csp_ext_src: 'https://dev.eduid.se https://www.dev.eduid.se'
|
||||
allow_ports:
|
||||
- 443
|
||||
letsencrypt_server: 'acme-c.dev.eduid.se'
|
||||
haproxy_imagetag: 'staging'
|
||||
405
fe-common/overlay/opt/frontend/config/common/default.vcl
Normal file
405
fe-common/overlay/opt/frontend/config/common/default.vcl
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
vcl 4.0;
|
||||
# Based on: https://github.com/mattiasgeniar/varnish-4.0-configuration-templates/blob/master/default.vcl
|
||||
|
||||
import std;
|
||||
import directors;
|
||||
|
||||
backend server1 { # Define one backend
|
||||
.host = "haproxy"; # IP or Hostname of backend
|
||||
.port = "1080"; # Port Apache or whatever is listening
|
||||
.max_connections = 300; # That's it
|
||||
|
||||
# .probe = {
|
||||
# #.url = "/"; # short easy way (GET /)
|
||||
# # We prefer to only do a HEAD /
|
||||
# .request =
|
||||
# "HEAD / HTTP/1.1"
|
||||
# "Host: localhost"
|
||||
# "Connection: close"
|
||||
# "User-Agent: Varnish Health Probe";
|
||||
#
|
||||
# .interval = 5s; # check the health of each backend every 5 seconds
|
||||
# .timeout = 1s; # timing out after 1 second.
|
||||
# .window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick
|
||||
# .threshold = 3;
|
||||
# }
|
||||
|
||||
.first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend?
|
||||
.connect_timeout = 5s; # How long to wait for a backend connection?
|
||||
.between_bytes_timeout = 2s; # How long to wait between bytes received from our backend?
|
||||
}
|
||||
|
||||
acl purge {
|
||||
# ACL we'll use later to allow purges
|
||||
"localhost";
|
||||
"127.0.0.1";
|
||||
"::1";
|
||||
}
|
||||
|
||||
sub vcl_init {
|
||||
# Called when VCL is loaded, before any requests pass through it.
|
||||
# Typically used to initialize VMODs.
|
||||
|
||||
new vdir = directors.round_robin();
|
||||
vdir.add_backend(server1);
|
||||
# vdir.add_backend(server...);
|
||||
# vdir.add_backend(servern);
|
||||
}
|
||||
|
||||
sub vcl_recv {
|
||||
# Called at the beginning of a request, after the complete request has been received and parsed.
|
||||
# Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable,
|
||||
# which backend to use.
|
||||
# also used to modify the request
|
||||
|
||||
set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
|
||||
|
||||
# Normalize the header, remove the port (in case you're testing this on various TCP ports)
|
||||
set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
|
||||
|
||||
# Remove the proxy header (see https://httpoxy.org/#mitigate-varnish)
|
||||
unset req.http.proxy;
|
||||
|
||||
# Normalize the query arguments
|
||||
set req.url = std.querysort(req.url);
|
||||
|
||||
# Allow purging
|
||||
if (req.method == "PURGE") {
|
||||
if (!client.ip ~ purge) { # purge is the ACL defined at the begining
|
||||
# Not from an allowed IP? Then die with an error.
|
||||
return (synth(405, "This IP is not allowed to send PURGE requests."));
|
||||
}
|
||||
# If you got this stage (and didn't error out above), purge the cached result
|
||||
return (purge);
|
||||
}
|
||||
|
||||
# Only deal with "normal" types
|
||||
if (req.method != "GET" &&
|
||||
req.method != "HEAD" &&
|
||||
req.method != "PUT" &&
|
||||
req.method != "POST" &&
|
||||
req.method != "TRACE" &&
|
||||
req.method != "OPTIONS" &&
|
||||
req.method != "PATCH" &&
|
||||
req.method != "DELETE") {
|
||||
/* Non-RFC2616 or CONNECT which is weird. */
|
||||
return (pipe);
|
||||
}
|
||||
|
||||
# Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
|
||||
if (req.http.Upgrade ~ "(?i)websocket") {
|
||||
return (pipe);
|
||||
}
|
||||
|
||||
# Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
|
||||
if (req.method != "GET" && req.method != "HEAD") {
|
||||
return (pass);
|
||||
}
|
||||
|
||||
# Some generic URL manipulation, useful for all templates that follow
|
||||
# First remove the Google Analytics added parameters, useless for our backend
|
||||
if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
|
||||
set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
|
||||
set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
|
||||
set req.url = regsub(req.url, "\?&", "?");
|
||||
set req.url = regsub(req.url, "\?$", "");
|
||||
}
|
||||
|
||||
# Strip hash, server doesn't need it.
|
||||
if (req.url ~ "\#") {
|
||||
set req.url = regsub(req.url, "\#.*$", "");
|
||||
}
|
||||
|
||||
# Strip a trailing ? if it exists
|
||||
if (req.url ~ "\?$") {
|
||||
set req.url = regsub(req.url, "\?$", "");
|
||||
}
|
||||
|
||||
# Some generic cookie manipulation, useful for all templates that follow
|
||||
# Remove the "has_js" cookie
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
|
||||
|
||||
# Remove any Google Analytics based cookies
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", "");
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
|
||||
|
||||
# Remove DoubleClick offensive cookies
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", "");
|
||||
|
||||
# Remove the Quant Capital cookies (added by some plugin, all __qca)
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
|
||||
|
||||
# Remove the AddThis cookies
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", "");
|
||||
|
||||
# Remove a ";" prefix in the cookie if present
|
||||
set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
|
||||
|
||||
# Are there cookies left with only spaces or that are empty?
|
||||
if (req.http.cookie ~ "^\s*$") {
|
||||
unset req.http.cookie;
|
||||
}
|
||||
|
||||
if (req.http.Cache-Control ~ "(?i)no-cache") {
|
||||
if (client.ip ~ purge) {
|
||||
# Ignore requests via proxy caches and badly behaved crawlers
|
||||
# like msnbot that send no-cache with every request.
|
||||
if (! (req.http.Via || req.http.User-Agent ~ "(?i)bot" || req.http.X-Purge)) {
|
||||
#set req.hash_always_miss = true; # Doesn't seems to refresh the object in the cache
|
||||
return(purge); # Couple this with restart in vcl_purge and X-Purge header to avoid loops
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Large static files are delivered directly to the end-user without
|
||||
# waiting for Varnish to fully read the file first.
|
||||
# Varnish 4 fully supports Streaming, so set do_stream in vcl_backend_response()
|
||||
if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
|
||||
unset req.http.Cookie;
|
||||
return (hash);
|
||||
}
|
||||
|
||||
# Remove all cookies for static files
|
||||
# A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
|
||||
# Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
|
||||
# Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
|
||||
if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
|
||||
unset req.http.Cookie;
|
||||
return (hash);
|
||||
}
|
||||
|
||||
# Send Surrogate-Capability headers to announce ESI support to backend
|
||||
set req.http.Surrogate-Capability = "key=ESI/1.0";
|
||||
|
||||
if (req.http.Authorization) {
|
||||
# Not cacheable by default
|
||||
return (pass);
|
||||
}
|
||||
|
||||
return (hash);
|
||||
}
|
||||
|
||||
sub vcl_pipe {
|
||||
# Called upon entering pipe mode.
|
||||
# In this mode, the request is passed on to the backend, and any further data from both the client
|
||||
# and backend is passed on unaltered until either end closes the connection. Basically, Varnish will
|
||||
# degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode,
|
||||
# no other VCL subroutine will ever get called after vcl_pipe.
|
||||
|
||||
# Note that only the first request to the backend will have
|
||||
# X-Forwarded-For set. If you use X-Forwarded-For and want to
|
||||
# have it set for all requests, make sure to have:
|
||||
# set bereq.http.connection = "close";
|
||||
# here. It is not set by default as it might break some broken web
|
||||
# applications, like IIS with NTLM authentication.
|
||||
|
||||
# set bereq.http.Connection = "Close";
|
||||
|
||||
# Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
|
||||
if (req.http.upgrade) {
|
||||
set bereq.http.upgrade = req.http.upgrade;
|
||||
}
|
||||
|
||||
return (pipe);
|
||||
}
|
||||
|
||||
sub vcl_pass {
|
||||
# Called upon entering pass mode. In this mode, the request is passed on to the backend, and the
|
||||
# backend's response is passed on to the client, but is not entered into the cache. Subsequent
|
||||
# requests submitted over the same client connection are handled normally.
|
||||
|
||||
# return (pass);
|
||||
}
|
||||
|
||||
# The data on which the hashing will take place
|
||||
sub vcl_hash {
|
||||
# Called after vcl_recv to create a hash value for the request. This is used as a key
|
||||
# to look up the object in Varnish.
|
||||
|
||||
hash_data(req.url);
|
||||
|
||||
if (req.http.host) {
|
||||
hash_data(req.http.host);
|
||||
} else {
|
||||
hash_data(server.ip);
|
||||
}
|
||||
|
||||
# hash cookies for requests that have them
|
||||
if (req.http.Cookie) {
|
||||
hash_data(req.http.Cookie);
|
||||
}
|
||||
}
|
||||
|
||||
sub vcl_hit {
|
||||
# Called when a cache lookup is successful.
|
||||
|
||||
if (obj.ttl >= 0s) {
|
||||
# A pure unadultered hit, deliver it
|
||||
return (deliver);
|
||||
}
|
||||
|
||||
# https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
|
||||
# When several clients are requesting the same page Varnish will send one request to the backend and place the others on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does this automatically.
|
||||
# If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache beyond their TTL and to serve the waiting requests somewhat stale content.
|
||||
|
||||
# if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
|
||||
# return (deliver);
|
||||
# } else {
|
||||
# return (miss);
|
||||
# }
|
||||
|
||||
# We have no fresh fish. Lets look at the stale ones.
|
||||
if (std.healthy(req.backend_hint)) {
|
||||
# Backend is healthy. Limit age to 10s.
|
||||
if (obj.ttl + 10s > 0s) {
|
||||
#set req.http.grace = "normal(limited)";
|
||||
return (deliver);
|
||||
} else {
|
||||
# No candidate for grace. Fetch a fresh object.
|
||||
return(miss);
|
||||
}
|
||||
} else {
|
||||
# backend is sick - use full grace
|
||||
if (obj.ttl + obj.grace > 0s) {
|
||||
#set req.http.grace = "full";
|
||||
return (deliver);
|
||||
} else {
|
||||
# no graced object.
|
||||
return (miss);
|
||||
}
|
||||
}
|
||||
|
||||
# fetch & deliver once we get the result
|
||||
return (miss); # Dead code, keep as a safeguard
|
||||
}
|
||||
|
||||
sub vcl_miss {
|
||||
# Called after a cache lookup if the requested document was not found in the cache. Its purpose
|
||||
# is to decide whether or not to attempt to retrieve the document from the backend, and which
|
||||
# backend to use.
|
||||
|
||||
return (fetch);
|
||||
}
|
||||
|
||||
# Handle the HTTP request coming from our backend
|
||||
sub vcl_backend_response {
|
||||
# Called after the response headers has been successfully retrieved from the backend.
|
||||
|
||||
# Pause ESI request and remove Surrogate-Control header
|
||||
if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
|
||||
unset beresp.http.Surrogate-Control;
|
||||
set beresp.do_esi = true;
|
||||
}
|
||||
|
||||
# Enable cache for all static files
|
||||
# The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
|
||||
# Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
|
||||
if (bereq.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
|
||||
unset beresp.http.set-cookie;
|
||||
}
|
||||
|
||||
# Large static files are delivered directly to the end-user without
|
||||
# waiting for Varnish to fully read the file first.
|
||||
# Varnish 4 fully supports Streaming, so use streaming here to avoid locking.
|
||||
if (bereq.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
|
||||
unset beresp.http.set-cookie;
|
||||
set beresp.do_stream = true; # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects
|
||||
}
|
||||
|
||||
# Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
|
||||
# This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
|
||||
# A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
|
||||
# This may need finetuning on your setup.
|
||||
#
|
||||
# To prevent accidental replace, we only filter the 301/302 redirects for now.
|
||||
if (beresp.status == 301 || beresp.status == 302) {
|
||||
set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
|
||||
}
|
||||
|
||||
# Set 2min cache if unset for static files
|
||||
if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
|
||||
set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend
|
||||
set beresp.uncacheable = true;
|
||||
return (deliver);
|
||||
}
|
||||
|
||||
# Don't cache 50x responses
|
||||
if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
|
||||
return (abandon);
|
||||
}
|
||||
|
||||
# Allow stale content, in case the backend goes down.
|
||||
# make Varnish keep all objects for 6 hours beyond their TTL
|
||||
set beresp.grace = 6h;
|
||||
|
||||
return (deliver);
|
||||
}
|
||||
|
||||
# The routine when we deliver the HTTP request to the user
|
||||
# Last chance to modify headers that are sent to the client
|
||||
sub vcl_deliver {
|
||||
# Called before a cached object is delivered to the client.
|
||||
|
||||
if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
|
||||
set resp.http.X-Cache = "HIT";
|
||||
} else {
|
||||
set resp.http.X-Cache = "MISS";
|
||||
}
|
||||
|
||||
# Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object
|
||||
# and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details.
|
||||
# So take hits with a grain of salt
|
||||
set resp.http.X-Cache-Hits = obj.hits;
|
||||
|
||||
# Remove some headers: PHP version
|
||||
unset resp.http.X-Powered-By;
|
||||
|
||||
# Remove some headers: Apache version & OS
|
||||
unset resp.http.Server;
|
||||
unset resp.http.X-Drupal-Cache;
|
||||
unset resp.http.X-Varnish;
|
||||
unset resp.http.Via;
|
||||
unset resp.http.Link;
|
||||
unset resp.http.X-Generator;
|
||||
|
||||
return (deliver);
|
||||
}
|
||||
|
||||
sub vcl_purge {
|
||||
# Only handle actual PURGE HTTP methods, everything else is discarded
|
||||
if (req.method != "PURGE") {
|
||||
# restart request
|
||||
set req.http.X-Purge = "Yes";
|
||||
return(restart);
|
||||
}
|
||||
}
|
||||
|
||||
sub vcl_synth {
|
||||
if (resp.status == 720) {
|
||||
# We use this special error status 720 to force redirects with 301 (permanent) redirects
|
||||
# To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
|
||||
set resp.http.Location = resp.reason;
|
||||
set resp.status = 301;
|
||||
return (deliver);
|
||||
} elseif (resp.status == 721) {
|
||||
# And we use error status 721 to force redirects with a 302 (temporary) redirect
|
||||
# To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
|
||||
set resp.http.Location = resp.reason;
|
||||
set resp.status = 302;
|
||||
return (deliver);
|
||||
}
|
||||
|
||||
return (deliver);
|
||||
}
|
||||
|
||||
|
||||
sub vcl_fini {
|
||||
# Called when VCL is discarded only after all requests have exited the VCL.
|
||||
# Typically used to clean up VMODs.
|
||||
|
||||
return (ok);
|
||||
}
|
||||
104
fe-common/overlay/opt/frontend/config/common/haproxy_base.j2
Normal file
104
fe-common/overlay/opt/frontend/config/common/haproxy_base.j2
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# haproxy for SUNET frontend load balancer nodes.
|
||||
#
|
||||
{% from "common/haproxy_macros.j2" import output_backends %}
|
||||
|
||||
{% block global %}
|
||||
global
|
||||
log /dev/log local0
|
||||
|
||||
daemon
|
||||
maxconn 256
|
||||
stats socket /var/run/haproxy-control/stats mode 600
|
||||
#server-state-file /tmp/server_state
|
||||
|
||||
user haproxy
|
||||
group haproxy
|
||||
|
||||
# Default SSL material locations
|
||||
ca-base /etc/ssl/certs
|
||||
crt-base /etc/ssl/private
|
||||
|
||||
ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
|
||||
ssl-default-bind-options no-sslv3
|
||||
tune.ssl.default-dh-param 2048
|
||||
|
||||
spread-checks 20
|
||||
{% endblock global %}
|
||||
|
||||
|
||||
{% block defaults %}
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option dontlognull
|
||||
option redispatch
|
||||
option forwardfor
|
||||
# funny looking values because recommendation is to have these slightly
|
||||
# above mulitples of three seconds to play nice with TCP resend timers
|
||||
timeout check 5s
|
||||
timeout connect 4s
|
||||
timeout client 17s
|
||||
timeout server 17s
|
||||
timeout http-request 5s
|
||||
balance roundrobin
|
||||
{% endblock defaults %}
|
||||
|
||||
{% block stats %}
|
||||
frontend LB-http
|
||||
# expose stats info over HTTP to exabgp
|
||||
bind 127.0.0.1:9000
|
||||
http-request set-log-level silent
|
||||
default_backend LB
|
||||
|
||||
backend LB
|
||||
stats enable
|
||||
#stats hide-version
|
||||
stats uri /haproxy_stats
|
||||
{% endblock stats %}
|
||||
|
||||
|
||||
{% block global_backends %}
|
||||
{% if letsencrypt_server is defined %}
|
||||
backend letsencrypt_{{ letsencrypt_server }}
|
||||
server letsencrypt_{{ letsencrypt_server }} {{ letsencrypt_server }}:81
|
||||
{% else %}
|
||||
# letsencrypt_server not defined
|
||||
{% endif %}
|
||||
{% endblock global_backends %}
|
||||
|
||||
|
||||
{% block https_everything %}
|
||||
#
|
||||
# Redirect _everything_ to HTTPS
|
||||
frontend http-frontend
|
||||
bind 0.0.0.0:80
|
||||
bind :::80
|
||||
|
||||
redirect scheme https code 301 if !{ ssl_fc } ! { path_beg /.well-known/acme-challenge/ }
|
||||
{% if letsencrypt_server is defined %}
|
||||
use_backend letsencrypt_{{ letsencrypt_server }} if { path_beg /.well-known/acme-challenge/ }
|
||||
{% else %}
|
||||
# letsencrypt_server not defined
|
||||
{% endif %}
|
||||
{% endblock https_everything %}
|
||||
|
||||
#
|
||||
# Frontend section
|
||||
#
|
||||
{% block frontend %}
|
||||
{% endblock frontend %}
|
||||
|
||||
|
||||
#
|
||||
# Backend section
|
||||
#
|
||||
{% block pre_backend %}
|
||||
{% endblock pre_backend %}
|
||||
|
||||
{% block backend %}
|
||||
{{ output_backends(backends, config=['cookie SERVERID insert indirect nocache']) }}
|
||||
|
||||
backend failpage
|
||||
server failpage 0.0.0.0:82 backup
|
||||
{% endblock backend %}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
#
|
||||
# Macros
|
||||
#
|
||||
|
||||
{%- macro bind_ip_tls(bind_ips, port, tls_cert) -%}
|
||||
{%- for ip in bind_ips %}
|
||||
bind {{ ip }}:{{ port }} ssl crt {{ tls_cert }}
|
||||
{%- endfor %}
|
||||
{%- endmacro %}
|
||||
|
||||
|
||||
{%- macro web_security_options(list) -%}
|
||||
{%- for this in list %}
|
||||
{%- if this == 'no_frames' %}
|
||||
# Do not allow rendering the site within an frame, which prevents clickjacking.
|
||||
http-response set-header X-Frame-Options "DENY"
|
||||
|
||||
{% endif %}
|
||||
{%- if this == 'block_xss' %}
|
||||
# Enable browser supplied XSS-protection, even if has been turned off.
|
||||
# If XSS is detected by the browser, block it instead of trying to sanitize it.
|
||||
http-response set-header X-XSS-Protection "1; mode=block"
|
||||
|
||||
{% endif %}
|
||||
{%- if this == 'hsts' %}
|
||||
# 20 years in seconds is 630720000 (86400 * 365 * 20)
|
||||
http-response set-header Strict-Transport-Security "max-age=630720000"
|
||||
|
||||
{% endif %}
|
||||
{%- if this == 'no_sniff' %}
|
||||
# Prevent MIME-confusion attacks that can lead to e.g. XSS
|
||||
http-response set-header X-Content-Type-Options "nosniff"
|
||||
|
||||
{% endif %}
|
||||
{%- if this == 'no_cache' %}
|
||||
# The information is intended for a single user and must not
|
||||
# be cached by a shared cache and should always be revalidated.
|
||||
http-response set-header Cache-Control "no-cache, no-store, must-revalidate"
|
||||
http-response set-header Pragma "no-cache"
|
||||
http-response set-header Expires "0"
|
||||
|
||||
{% endif %}
|
||||
{%- endfor %}
|
||||
{%- endmacro %}
|
||||
|
||||
|
||||
{%- macro acme_challenge(letsencrypt_server) -%}
|
||||
{%- if letsencrypt_server is defined %}
|
||||
use_backend letsencrypt_{{ letsencrypt_server }} if { path_beg /.well-known/acme-challenge/ }
|
||||
{%- else %}
|
||||
# No letsencrypt_server specified
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
|
||||
{%- macro csp(data) -%}
|
||||
# Content Security Policy
|
||||
http-response set-header Content-Security-Policy "{{ data|join('; ') }}"
|
||||
{%- endmacro %}
|
||||
|
||||
{%- macro output_backends(backends, config=[], server_args='') -%}
|
||||
{% if backends is defined %}
|
||||
{%- for this in backends %}
|
||||
backend {{ this.name }}
|
||||
{{ config|join('\n ') }}
|
||||
{%- for server in this.servers %}
|
||||
{%- if server.server_args is defined %}
|
||||
{%- set server_args = server.server_args %}
|
||||
{%- endif %}
|
||||
{% if server is defined %}
|
||||
server {{ server.server }}_{{ server.address_family }} {{ server.ip }}:{{ server.port }} {{ server_args }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endfor %}
|
||||
{% else %}
|
||||
# No backends found in context
|
||||
{% endif %}
|
||||
{%- endmacro %}
|
||||
98
fe-common/overlay/opt/frontend/scripts/configure-container-network
Executable file
98
fe-common/overlay/opt/frontend/scripts/configure-container-network
Executable file
|
|
@ -0,0 +1,98 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Add all anycasted IP addresses for a frontend instance to it's network namespace.
|
||||
#
|
||||
# The network namespace used is the one of the haproxy container for the instance
|
||||
# specified as the only argument.
|
||||
#
|
||||
# This script sets up a virtual ethernet 'cable' into the container namespace.
|
||||
# The outside end is called e.g. www0 for instance www, and the inside end is
|
||||
# always called sarimner0.
|
||||
#
|
||||
# The haproxy-start.sh script waits for sarimner0 to come up before actually
|
||||
# starting haproxy.
|
||||
#
|
||||
|
||||
INSTANCE=$1
|
||||
|
||||
if [[ ! $INSTANCE ]]; then
|
||||
echo "Syntax: ${0} instance"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPTSDIR=$(dirname $0)
|
||||
SITE_NAME=$(${SCRIPTSDIR}/frontend-config --instance ${INSTANCE} print_site_name)
|
||||
if [[ ! ${SITE_NAME} ]]; then
|
||||
echo "$0: Could not get site_name for instance ${INSTANCE} using ${SCRIPTSDIR}/frontend-config"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONTAINER="${INSTANCE}_haproxy_1"
|
||||
for retry in $(seq 20); do
|
||||
DOCKERPID=$(docker inspect '--format={{ .State.Pid }}' ${CONTAINER})
|
||||
if [[ $DOCKERPID && $DOCKERPID != 0 ]]; then
|
||||
break
|
||||
fi
|
||||
echo "$0: Container ${CONTAINER} not found (attempt ${retry}/20)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [[ ! $DOCKERPID || $DOCKERPID == 0 ]]; then
|
||||
echo "$0: Could not find PID of docker container ${CONTAINER}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NSPID=${DOCKERPID}
|
||||
|
||||
mkdir -p /var/run/netns
|
||||
rm -f /var/run/netns/${INSTANCE}
|
||||
ln -s /proc/${NSPID}/ns/net /var/run/netns/${INSTANCE}
|
||||
|
||||
echo "Container ${CONTAINER} has pid ${DOCKERPID} - symlinking /var/run/netns/${INSTANCE} to /proc/${NSPID}/ns/net"
|
||||
|
||||
VETHHOST="${INSTANCE}"
|
||||
VETHCONTAINER="ve1${INSTANCE}"
|
||||
|
||||
set -x
|
||||
|
||||
# Enable IPv6 forwarding. Should ideally be done more selectively, but...
|
||||
sysctl net.ipv6.conf.all.forwarding=1
|
||||
|
||||
# Add a pair of virtual ethernet interfaces (think of them as a virtual cross-over ethernet cable)
|
||||
ip link add name ${VETHHOST} mtu 1500 type veth peer name ${VETHCONTAINER} mtu 1500
|
||||
ip link set ${VETHHOST} master br-${INSTANCE}
|
||||
ip link set ${VETHHOST} up
|
||||
|
||||
# Move one end of the virtual ethernet cable inside the network namespace of the docker container
|
||||
ip link set ${VETHCONTAINER} netns ${INSTANCE} || {
|
||||
echo "$0: FAILED to configure namespace, did ${CONTAINER} (pid ${DOCKERPID}) die?"
|
||||
exit 1
|
||||
}
|
||||
ip netns exec ${INSTANCE} ip link set ${VETHCONTAINER} name sarimner0
|
||||
|
||||
# Docker likes to disable IPv6
|
||||
ip netns exec ${INSTANCE} sysctl net.ipv6.conf.sarimner0.disable_ipv6=0
|
||||
# DAD interferes with haproxy's first bind() of the IPv6 addresses,
|
||||
# and should really not be needed inside the namespace
|
||||
ip netns exec ${INSTANCE} sysctl net.ipv6.conf.sarimner0.accept_dad=0
|
||||
# Allow bind to IP address before it is configured.
|
||||
# XXX Disabled since I can't decide if that would be a bug or a feature in this case.
|
||||
# ip netns exec ${INSTANCE} sysctl net.ipv4.ip_nonlocal_bind=1
|
||||
# ip netns exec ${INSTANCE} sysctl net.ipv6.ip_nonlocal_bind=1
|
||||
|
||||
# Add IPv6 default gateway
|
||||
#sysctl net.ipv6.conf.${VETHHOST}.accept_dad=0
|
||||
v6gw=$(ip -6 addr list br-${INSTANCE} | awk '/inet6/{print $2}' | head -1 | awk -F / '{print $1}')
|
||||
if [[ $v6gw ]]; then
|
||||
ip netns exec ${INSTANCE} ip -6 route add default via ${v6gw} dev sarimner0
|
||||
else
|
||||
echo "Can't set up IPv6 routing from container, device ${VETHHOST} has no IPv6 address"
|
||||
fi
|
||||
|
||||
# Add IP addresses to the network namespace of the docker container
|
||||
for IP in $(${SCRIPTSDIR}/frontend-config --instance ${INSTANCE} print_ips); do
|
||||
ip netns exec ${INSTANCE} ip addr add ${IP} dev sarimner0
|
||||
ip route add ${IP} dev br-${INSTANCE}
|
||||
done
|
||||
|
||||
ip netns exec ${INSTANCE} ip link set sarimner0 up
|
||||
423
fe-common/overlay/opt/frontend/scripts/frontend-config
Executable file
423
fe-common/overlay/opt/frontend/scripts/frontend-config
Executable file
|
|
@ -0,0 +1,423 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2018 SUNET. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are
|
||||
# permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
|
||||
# of conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY SUNET ``AS IS'' AND ANY EXPRESS OR IMPLIED
|
||||
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SUNET OR
|
||||
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
# The views and conclusions contained in the software and documentation are those of the
|
||||
# authors and should not be interpreted as representing official policies, either expressed
|
||||
# or implied, of SUNET.
|
||||
#
|
||||
# Author: Fredrik Thulin <fredrik@thulin.net>
|
||||
#
|
||||
|
||||
"""
|
||||
Toolbox for everything that needs to read the instance YAML config file.
|
||||
|
||||
The idea is to not have a lot of different scripts reading that config file,
|
||||
to make it easier to update the YAML config file format going forward.
|
||||
|
||||
Main actions:
|
||||
|
||||
- Print things in the YAML config file. Some things are handled specially,
|
||||
like 'print_ips' that extracts frontend IP addresses and make them convenient
|
||||
to work with from shell scripts.
|
||||
|
||||
- Print haproxy config. Creates a Jinja2 context from the YAML file and generates a
|
||||
haproxy config from a template.
|
||||
|
||||
- Print exabgp announce messages. Called whenever a status change is detected
|
||||
for the configured backends.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import glob
|
||||
import time
|
||||
import yaml
|
||||
import pprint
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
import logging.handlers
|
||||
|
||||
logger = None
|
||||
|
||||
_defaults = {'syslog': False,
|
||||
'debug': False,
|
||||
'config_dir': '/opt/frontend/config',
|
||||
'backends_dir': '/opt/frontend/api/backends',
|
||||
'max_age': 600,
|
||||
}
|
||||
|
||||
|
||||
def parse_args(defaults):
|
||||
parser = argparse.ArgumentParser(description = 'SUNET frontend config toolbox',
|
||||
add_help = True,
|
||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
|
||||
# Positional arguments
|
||||
parser.add_argument('actions',
|
||||
metavar='ACTION',
|
||||
nargs='+',
|
||||
help='Actions to perform',
|
||||
)
|
||||
|
||||
# Optional arguments
|
||||
parser.add_argument('--config_dir',
|
||||
dest = 'config_dir',
|
||||
metavar = 'DIR', type = str,
|
||||
default = defaults['config_dir'],
|
||||
help = 'Base directory for configuration data',
|
||||
)
|
||||
parser.add_argument('--backends_dir',
|
||||
dest = 'backends_dir',
|
||||
metavar = 'DIR', type = str,
|
||||
default = defaults['backends_dir'],
|
||||
help = 'Base directory for API backend registration data',
|
||||
)
|
||||
parser.add_argument('-c', '--config_fn',
|
||||
dest = 'config_fn',
|
||||
metavar = 'FILENAME', type = str,
|
||||
default = None,
|
||||
help = 'YAML config file to use',
|
||||
)
|
||||
parser.add_argument('--haproxy_template',
|
||||
dest = 'haproxy_template',
|
||||
metavar = 'FILENAME', type = str,
|
||||
default = None,
|
||||
help = 'haproxy.j2 file to use, relative to --config-dir',
|
||||
)
|
||||
parser.add_argument('--instance',
|
||||
dest = 'instance',
|
||||
metavar='NAME', type = str,
|
||||
default = None,
|
||||
help='Instance',
|
||||
)
|
||||
parser.add_argument('--max_age',
|
||||
dest = 'max_age',
|
||||
metavar = 'SECONDS', type = int,
|
||||
default = defaults['max_age'],
|
||||
help = 'Max backend file age to allow',
|
||||
)
|
||||
parser.add_argument('--fqdn',
|
||||
dest = 'fqdn',
|
||||
metavar='NAME', type = str,
|
||||
default = None,
|
||||
help='Host FQDN (for print_exabgp_announce)',
|
||||
)
|
||||
parser.add_argument('--status',
|
||||
dest = 'status',
|
||||
metavar='UP/DOWN', type = str,
|
||||
default = None,
|
||||
help='Current backend status (for print_exabgp_announce)',
|
||||
)
|
||||
|
||||
parser.add_argument('--debug',
|
||||
dest = 'debug',
|
||||
action = 'store_true', default = defaults['debug'],
|
||||
help = 'Enable debug operation',
|
||||
)
|
||||
parser.add_argument('--syslog',
|
||||
dest = 'syslog',
|
||||
action = 'store_true', default = defaults['syslog'],
|
||||
help = 'Enable syslog output',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def main(myname = 'frontend-config', args = None, logger_in = None, defaults = _defaults):
|
||||
if not args:
|
||||
args = parse_args(defaults)
|
||||
|
||||
global logger
|
||||
# initialize various components
|
||||
if logger_in:
|
||||
logger = logger_in
|
||||
else:
|
||||
# This is the root log level
|
||||
level = logging.INFO
|
||||
if args.debug:
|
||||
level = logging.DEBUG
|
||||
logging.basicConfig(level = level, stream = sys.stderr,
|
||||
format='%(asctime)s: %(name)s: %(threadName)s %(levelname)s %(message)s')
|
||||
logger = logging.getLogger(myname)
|
||||
# If stderr is not a TTY, change the log level of the StreamHandler (stream = sys.stderr above) to WARNING
|
||||
if not sys.stderr.isatty() and not args.debug:
|
||||
for this_h in logging.getLogger('').handlers:
|
||||
this_h.setLevel(logging.WARNING)
|
||||
if args.syslog:
|
||||
syslog_h = logging.handlers.SysLogHandler()
|
||||
formatter = logging.Formatter('%(name)s: %(levelname)s %(message)s')
|
||||
syslog_h.setFormatter(formatter)
|
||||
logger.addHandler(syslog_h)
|
||||
|
||||
config_fn = args.config_fn
|
||||
if not config_fn:
|
||||
if not args.instance:
|
||||
logger.error('Neither config file nor --instance supplied')
|
||||
return False
|
||||
config_fn = os.path.join(args.config_dir, args.instance, 'config.yml')
|
||||
|
||||
logger.debug('Loading configuration from {!r}'.format(config_fn))
|
||||
|
||||
with open(config_fn, 'r') as fd:
|
||||
config = yaml.safe_load(fd)
|
||||
|
||||
logger.debug('Config:\n{!s}'.format(pprint.pformat(config)))
|
||||
|
||||
for this in args.actions:
|
||||
logger.debug('Processing action {!r}'.format(this))
|
||||
if this.startswith('print_'):
|
||||
if this == 'print_ips':
|
||||
if not print_ips(config, args, logger):
|
||||
return False
|
||||
elif this == 'print_haproxy_config':
|
||||
if not print_haproxy_config(config, args, logger):
|
||||
return False
|
||||
elif this == 'print_exabgp_announce':
|
||||
if not print_exabgp_announce(config, args, logger):
|
||||
return False
|
||||
else:
|
||||
# Print generic value
|
||||
if not print_generic(this[6:], config, args, logger):
|
||||
return False
|
||||
else:
|
||||
sys.stderr.write('Unknown action {!r}\n'.format(this))
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def print_generic(what, config, args, logger):
|
||||
if what not in config:
|
||||
return False
|
||||
print(config[what])
|
||||
return True
|
||||
|
||||
|
||||
def print_ips(config, args, logger):
|
||||
if not 'frontends' in config:
|
||||
return False
|
||||
res = []
|
||||
for fe in config['frontends'].values():
|
||||
res += fe['ips']
|
||||
print('\n'.join(sorted(res)))
|
||||
return True
|
||||
|
||||
|
||||
def print_exabgp_announce(config, args, logger):
|
||||
if not args.fqdn:
|
||||
logger.error('Action print_exabgp_announce requires --fqdn')
|
||||
return False
|
||||
if not args.status:
|
||||
logger.error('Action print_exabgp_announce requires --status')
|
||||
return False
|
||||
frontends = _pref_sort_frontends(args.fqdn, sorted(config['frontends'].keys()))
|
||||
|
||||
bgp_pref = 1000
|
||||
is_up = True if args.status.startswith('UP') else False
|
||||
if not is_up:
|
||||
# fallback route preference
|
||||
bgp_pref = 100
|
||||
bgp = []
|
||||
count=1
|
||||
frontends.reverse()
|
||||
for fe in frontends:
|
||||
for ip in sorted(config['frontends'][fe]['ips']):
|
||||
if ':' in ip:
|
||||
ip_slash = ip + '/128'
|
||||
else:
|
||||
ip_slash = ip + '/32'
|
||||
|
||||
logger.debug('Site {}, address {}/{} -> announce {} with BGP preference {}'.format(
|
||||
config['site_name'], count, len(frontends), ip, bgp_pref))
|
||||
bgp.append('announce route {} next-hop self local-preference {}'.format(ip_slash, bgp_pref))
|
||||
count += 1
|
||||
if is_up:
|
||||
bgp_pref += 100
|
||||
else:
|
||||
bgp_pref += 10
|
||||
|
||||
print('\n'.join(sorted(bgp)))
|
||||
return bool(bgp)
|
||||
|
||||
|
||||
def _pref_sort_frontends(fqdn, frontends):
|
||||
"""
|
||||
Three load balancers named A, B and C should all announce
|
||||
service IP 1, 2 and 3 with different BGP MED (lower MED wins).
|
||||
|
||||
A should announce 1 with the lowest MED, and B should announce
|
||||
2 with the lowest MED.
|
||||
|
||||
Re-sort the input list of IPs to start with the one specified by the index.
|
||||
"""
|
||||
index = frontends.index(fqdn)
|
||||
return frontends[index:] + frontends[:index]
|
||||
|
||||
|
||||
def print_haproxy_config(config, args, logger):
|
||||
if not args.haproxy_template:
|
||||
logger.error('Action print_haproxy_config requires --haproxy_template')
|
||||
return False
|
||||
|
||||
if not args.instance:
|
||||
logger.error('Action print_haproxy_config requires --instance')
|
||||
return False
|
||||
|
||||
context = config
|
||||
|
||||
import pprint
|
||||
import jinja2
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
bind_ips = []
|
||||
for fe in config['frontends'].values():
|
||||
bind_ips += fe['ips']
|
||||
context['bind_ips'] = sorted(bind_ips)
|
||||
|
||||
context['backends'] = _load_backends(config, args, logger)
|
||||
|
||||
# remove things not meant for the haproxy template
|
||||
for this in ['allow_ports', 'frontends', 'frontend_template']:
|
||||
if this in context:
|
||||
del context[this]
|
||||
|
||||
logger.debug('Rendering haproxy template {} with context:\n{!s}'.format(args.haproxy_template, pprint.pformat(context)))
|
||||
|
||||
env = Environment(loader = FileSystemLoader(args.config_dir))
|
||||
template = env.get_template(args.haproxy_template)
|
||||
print(template.render(**context))
|
||||
return True
|
||||
|
||||
def _load_backends(config, args, logger):
|
||||
"""
|
||||
Load data written to disk by the API used by backends to register themselves.
|
||||
"""
|
||||
res = []
|
||||
for be in sorted(config['backends'].keys()):
|
||||
# be is 'default', 'testing' or similar
|
||||
if be.startswith('_'):
|
||||
continue
|
||||
|
||||
servers = []
|
||||
be_name = '{}__{}'.format(config['site_name'], be)
|
||||
for host, params in config['backends'][be].items():
|
||||
be_ips = params.pop('ips', [])
|
||||
for addr in be_ips:
|
||||
data = _load_backend_data(config['site_name'], addr, args, logger)
|
||||
logger.debug('Backend data for site {} addr {}: {}'.format(config['site_name'], addr, data))
|
||||
if data is None:
|
||||
logger.debug('Not adding backend {} - probably not registered'.format(addr))
|
||||
continue
|
||||
params.update(data)
|
||||
servers += [params]
|
||||
|
||||
res += [{'name': be_name,
|
||||
'servers': servers,
|
||||
}]
|
||||
return res
|
||||
|
||||
|
||||
def _load_backend_data(site_name, addr, args, logger):
|
||||
"""
|
||||
Load data about a registered backend from a file.
|
||||
|
||||
Return data matching what the haproxy.j2 template wants for 'servers':
|
||||
|
||||
{'name': 'server1.example.org',
|
||||
'ip': '192.0.2.1',
|
||||
'port': '443',
|
||||
'address_family': 'v4'
|
||||
}
|
||||
"""
|
||||
match = os.path.join(args.backends_dir, site_name, '*_{}.conf'.format(addr))
|
||||
fns = glob.glob(match)
|
||||
if not fns:
|
||||
logger.debug('Found no file(s) matching {!r}'.format(match))
|
||||
return None
|
||||
for this in fns:
|
||||
# Check file age before using this file
|
||||
if _get_fileage(this) > args.max_age:
|
||||
logger.info('Ignoring file {} that is older than max_age ({})'.format(this, args.max_age))
|
||||
continue
|
||||
|
||||
logger.debug('Found backend file {}'.format(this))
|
||||
params = _load_backend_file(this)
|
||||
if params.pop('action') == 'register':
|
||||
params['address_family'] = 'v4' if params.get('ip') else None
|
||||
if ':' in params.get('ip', ''):
|
||||
params['address_family'] = 'v6'
|
||||
return params
|
||||
|
||||
|
||||
def _get_fileage(filename):
|
||||
st = os.stat(filename)
|
||||
mtime = st.st_mtime
|
||||
real_now = int(time.time())
|
||||
return int(real_now - mtime)
|
||||
|
||||
|
||||
def _load_backend_file(filename):
|
||||
"""
|
||||
Load a backend registration file, created by the sunet-frontend-api.
|
||||
|
||||
Example file:
|
||||
|
||||
ACTION=register
|
||||
BACKEND=www.dev.eduid.se
|
||||
SERVER=www-fre-1.eduid.se
|
||||
REMOTE_IP=130.242.130.200
|
||||
PORT=443
|
||||
|
||||
:returns: dict with normalized keys and their values
|
||||
:rtype: dict
|
||||
"""
|
||||
res = {}
|
||||
fd = open(filename)
|
||||
for line in fd.readlines():
|
||||
while line.endswith('\n'):
|
||||
line = line[:-1]
|
||||
(head, sep, tail) = line.partition('=')
|
||||
if head and tail:
|
||||
key = head.lower()
|
||||
if key == 'remote_ip':
|
||||
key = 'ip'
|
||||
elif key == 'backend':
|
||||
key = 'name'
|
||||
res[key] = tail
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
progname = os.path.basename(sys.argv[0])
|
||||
res = main(progname)
|
||||
if res is True:
|
||||
sys.exit(0)
|
||||
if res is False:
|
||||
sys.exit(1)
|
||||
sys.exit(int(res))
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
47
fe-common/overlay/opt/frontend/scripts/generate-haproxy-config
Executable file
47
fe-common/overlay/opt/frontend/scripts/generate-haproxy-config
Executable file
|
|
@ -0,0 +1,47 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Generate haproxy configuration whenever a change to one of the inputs for said
|
||||
# generation is detected.
|
||||
#
|
||||
|
||||
cfgfile='/etc/haproxy/haproxy.cfg'
|
||||
|
||||
rm -f "${cfgfile}.new"
|
||||
|
||||
pid=0
|
||||
|
||||
term_handler() {
|
||||
echo "$0: Received SIGTERM, shutting down ${pid}"
|
||||
if [ $pid -ne 0 ]; then
|
||||
kill -SIGTERM "$pid"
|
||||
wait "$pid"
|
||||
fi
|
||||
exit 143; # 128 + 15 -- SIGTERM
|
||||
}
|
||||
|
||||
trap 'kill ${!}; term_handler' SIGTERM
|
||||
|
||||
while [ 1 ]; do
|
||||
/opt/frontend/scripts/frontend-config $* print_haproxy_config > /etc/haproxy/haproxy.cfg.new
|
||||
|
||||
changed=0
|
||||
if [ -s /etc/haproxy/haproxy.cfg.new ]; then
|
||||
cmp --quiet "${cfgfile}.new" "${cfgfile}" || changed=1
|
||||
if [ $changed -ne 0 ]; then
|
||||
echo "haproxy config changed:";
|
||||
diff -u "${cfgfile}" "${cfgfile}.new"
|
||||
# this mv will inotify-trigger the autoreload.sh in the haproxy container to reload haproxy
|
||||
mv "${cfgfile}.new" "${cfgfile}"
|
||||
else
|
||||
echo "haproxy config did not change"
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 1 # spin control
|
||||
|
||||
# The only things volume-mounted into these directories in the container where this runs
|
||||
# should be specific to this instance, so we're not triggering off updates to other instances
|
||||
inotifywait -q -r -e moved_to -e close_write /opt/frontend/api/backends /opt/frontend/config &
|
||||
pid=${!}
|
||||
wait $pid
|
||||
done
|
||||
98
fe-common/overlay/opt/frontend/scripts/haproxy-start.sh
Executable file
98
fe-common/overlay/opt/frontend/scripts/haproxy-start.sh
Executable file
|
|
@ -0,0 +1,98 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Start script for haproxy container, managing startup process and automatic reload
|
||||
# on config change (detected using inotify events triggering on MOVED_TO).
|
||||
#
|
||||
|
||||
HAPROXYCFG=${HAPROXYCFG-'/etc/haproxy/haproxy.cfg'}
|
||||
HAPROXYWAITIF=${HAPROXYWAITIF-'20'}
|
||||
HAPROXYWAITCFG=${HAPROXYWAITCFG-'10'}
|
||||
HAPROXYWAITCONTAINER=${HAPROXYWAITCONTAINER-'10'}
|
||||
|
||||
if [[ $WAIT_FOR_INTERFACE ]]; then
|
||||
for i in $(seq ${HAPROXYWAITIF}); do
|
||||
ip link ls dev "$WAIT_FOR_INTERFACE" | grep -q 'state UP' && break
|
||||
echo "$0: Waiting for interface ${WAIT_FOR_INTERFACE} (${i}/${HAPROXYWAITIF})"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! ip link ls dev "$WAIT_FOR_INTERFACE" | grep -q 'state UP'; then
|
||||
echo "$0: Interface ${WAIT_FOR_INTERFACE} not found after ${HAPROXYWAITIF} seconds"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$0: Interface ${WAIT_FOR_INTERFACE} is UP:"
|
||||
ip addr list "$WAIT_FOR_INTERFACE"
|
||||
fi
|
||||
|
||||
for i in $(seq ${HAPROXYWAITCFG}); do
|
||||
test -f "${HAPROXYCFG}" && break
|
||||
echo "$0: Waiting for haproxy config file ${HAPROXYCFG} (${i}/${HAPROXYWAITCFG})"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ ! -f "${HAPROXYCFG}" ]; then
|
||||
echo "$0: haproxy config not found after ${HAPROXYWAITCFG} seconds: ${HAPROXYCFG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $WAIT_FOR_CONTAINER ]]; then
|
||||
seen=0
|
||||
for i in $(seq ${HAPROXYWAITCONTAINER}); do
|
||||
ping -c 1 $WAIT_FOR_CONTAINER > /dev/null 2>&1 && seen=1
|
||||
test $seen == 1 && break
|
||||
echo "$0: Waiting for container ${WAIT_FOR_CONTAINER} to appear (${i}/${HAPROXYWAITCONTAINER})"
|
||||
sleep 1
|
||||
done
|
||||
if [[ $seen != 1 ]]; then
|
||||
echo "$0: Host ${WAIT_FOR_CONTAINER} not present after ${HAPROXYWAITCONTAINER} seconds"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
echo "$0: Checking config: ${HAPROXYCFG}"
|
||||
|
||||
/usr/sbin/haproxy -c -f "${HAPROXYCFG}"
|
||||
|
||||
echo "$0: Config ${HAPROXYCFG} checked OK, starting haproxy-systemd-wrapper"
|
||||
/usr/sbin/haproxy-systemd-wrapper -p /run/haproxy.pid -f "${HAPROXYCFG}" &
|
||||
pid=$!
|
||||
pid2=0
|
||||
|
||||
term_handler() {
|
||||
echo "$0: Received SIGTERM, shutting down ${pid}, ${pid2}"
|
||||
if [ $pid -ne 0 ]; then
|
||||
kill -SIGTERM "$pid"
|
||||
wait "$pid"
|
||||
fi
|
||||
if [ $pid2 -ne 0 ]; then
|
||||
kill -SIGTERM "$pid2"
|
||||
wait "$pid2"
|
||||
fi
|
||||
exit 143; # 128 + 15 -- SIGTERM
|
||||
}
|
||||
|
||||
trap 'term_handler' SIGTERM
|
||||
|
||||
|
||||
while [ 1 ]; do
|
||||
echo "$0: Waiting for ${HAPROXYCFG} to be moved-to"
|
||||
|
||||
# Block until an inotify event says that the config file was replaced
|
||||
inotifywait -q -e moved_to "${HAPROXYCFG}" &
|
||||
pid2=$!
|
||||
wait $pid2
|
||||
|
||||
echo "$0: Move-to event triggered, checking config: ${HAPROXYCFG}"
|
||||
config_ok=1
|
||||
/usr/sbin/haproxy -c -f "${HAPROXYCFG}" || config_ok=0
|
||||
if [ $config_ok = 1 ]; then
|
||||
echo "$0: Config ${HAPROXYCFG} checked OK, gracefully restarting haproxy-systemd-wrapper"
|
||||
/usr/sbin/haproxy $* -p /run/haproxy.pid -f "${HAPROXYCFG}" -sf `cat /run/haproxy.pid`
|
||||
echo "$0: haproxy gracefully reloaded"
|
||||
else
|
||||
echo "$0: Config ${HAPROXYCFG} NOT OK"
|
||||
fi
|
||||
sleep 1 # spin control
|
||||
done
|
||||
304
fe-common/overlay/opt/frontend/scripts/haproxy-status
Executable file
304
fe-common/overlay/opt/frontend/scripts/haproxy-status
Executable file
|
|
@ -0,0 +1,304 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2018 SUNET. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are
|
||||
# permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
|
||||
# of conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY SUNET ``AS IS'' AND ANY EXPRESS OR IMPLIED
|
||||
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SUNET OR
|
||||
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
# The views and conclusions contained in the software and documentation are those of the
|
||||
# authors and should not be interpreted as representing official policies, either expressed
|
||||
# or implied, of SUNET.
|
||||
#
|
||||
# Author: Fredrik Thulin <fredrik@thulin.net>
|
||||
#
|
||||
|
||||
"""
|
||||
Check that haproxy backends are up (up as defined in the 'site' argument string format)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import socket
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
from logging.handlers import SysLogHandler
|
||||
|
||||
class Site(object):
|
||||
""" Wrapper object for parsed haproxy status data """
|
||||
def __init__(self):
|
||||
self.frontend = None
|
||||
self.backend = None
|
||||
self.groups = {}
|
||||
|
||||
def __repr__(self):
|
||||
return 'Site(frontend={!r}, backend={!r}, groups={!r})'.format(
|
||||
self.frontend, self.backend, sorted(self.groups))
|
||||
|
||||
def set_status(self, group, label, value):
|
||||
if label == 'FRONTEND':
|
||||
self.frontend = value
|
||||
elif label == 'BACKEND':
|
||||
self.backend = value
|
||||
else:
|
||||
if group not in self.groups:
|
||||
self.groups[group] = {}
|
||||
self.groups[group][label] = value
|
||||
|
||||
|
||||
|
||||
logger = None
|
||||
|
||||
_defaults = {'stats_url': 'http://127.0.0.1:9000/haproxy_stats;csv',
|
||||
'syslog': False,
|
||||
'debug': False,
|
||||
'interface': 'lo',
|
||||
}
|
||||
|
||||
class HAProxyStatusError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def parse_args(defaults):
|
||||
parser = argparse.ArgumentParser(description = 'haproxy status checker',
|
||||
add_help = True,
|
||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
|
||||
parser.add_argument('site',
|
||||
nargs='+',
|
||||
metavar='STR',
|
||||
help='Site to check, in key-value format (e.g. "site=www.dev.eduid.se; min_up=2")',
|
||||
)
|
||||
parser.add_argument('--debug',
|
||||
dest = 'debug',
|
||||
action = 'store_true', default = defaults['debug'],
|
||||
help = 'Enable debug operation',
|
||||
)
|
||||
parser.add_argument('--syslog',
|
||||
dest = 'syslog',
|
||||
action = 'store_true', default = defaults['syslog'],
|
||||
help = 'Enable syslog output',
|
||||
)
|
||||
parser.add_argument('--stats_url',
|
||||
dest = 'stats_url',
|
||||
default = defaults['stats_url'],
|
||||
help = 'haproxy stats URL (CSV format)',
|
||||
metavar = 'URL',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def haproxy_execute(cmd, args):
|
||||
if args.stats_url.startswith('http'):
|
||||
import requests
|
||||
|
||||
logger.debug('Fetching haproxy stats from {}'.format(args.stats_url))
|
||||
try:
|
||||
data = requests.get(args.stats_url).text
|
||||
except requests.exceptions.ConnectionError as exc:
|
||||
raise HAProxyStatusError('Failed fetching status from {}: {}'.format(args.stats_url, exc))
|
||||
else:
|
||||
socket_fn = args.stats_url
|
||||
if socket_fn.startswith('file://'):
|
||||
socket_fn = socket_fn[len('file://'):]
|
||||
logger.debug('opening AF_UNIX socket {} for command "{}"'.format(socket_fn, cmd))
|
||||
try:
|
||||
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
client.connect(socket_fn)
|
||||
client.send(cmd + '\n')
|
||||
except Exception as exc:
|
||||
logger.error('Failed sending command to socket {}: {}'.format(socket_fn, exc))
|
||||
return None
|
||||
|
||||
data = ''
|
||||
while True:
|
||||
this = client.recv(1)
|
||||
if not this:
|
||||
break
|
||||
data += this
|
||||
|
||||
logger.debug('haproxy result: {}'.format(data))
|
||||
return data
|
||||
|
||||
|
||||
def get_status(args):
|
||||
"""
|
||||
haproxy 'show stat' returns _a lot_ of different metrics for each frontend and backend
|
||||
in the system. Parse the returned CSV data and return the 'status' value for
|
||||
frontends and backends, example:
|
||||
|
||||
{'app.example.org': Site(frontend='OPEN',
|
||||
backend='UP',
|
||||
groups={'default': {
|
||||
'dash-fre-1.eduid.se_v4': 'UP',
|
||||
'dash-tug-1.eduid.se_v4': 'UP'
|
||||
'failpage': 'no check'},
|
||||
'new': {
|
||||
'apps-tug-1.eduid.se_v4': 'UP',
|
||||
'apps-fre-1.eduid.se_v4': 'UP',
|
||||
'failpage': 'no check'}}
|
||||
),
|
||||
...
|
||||
|
||||
:param args:
|
||||
:return: Status dict as detailed above
|
||||
:rtype: dict
|
||||
"""
|
||||
data = haproxy_execute('show stat', args)
|
||||
if not data:
|
||||
return None
|
||||
lines = data.split('\n')
|
||||
if not lines[0].startswith('# '):
|
||||
logger.error('Unknown status response from haproxy: {}'.format(data))
|
||||
# The first line is the legend, e.g.
|
||||
# # pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,...,status,...
|
||||
fields = lines[0][2:].split(',')
|
||||
if len(lines) < 2:
|
||||
logger.warning('haproxy did not return status for any backends: {}'.format(data))
|
||||
return None
|
||||
res = {}
|
||||
# parse all the lines with real data
|
||||
for line in lines[1:]:
|
||||
if not line:
|
||||
continue
|
||||
values = line.split(',')
|
||||
if len(values) != len(fields):
|
||||
logger.warning('Values ({}) does not match legend ({}): {}'.format(len(values), len(fields), line))
|
||||
continue
|
||||
site = values[0]
|
||||
group = 'default'
|
||||
logger.debug('processing site {!r}'.format(site))
|
||||
if '__' in site:
|
||||
site, group = site.split('__')
|
||||
label = values[1]
|
||||
|
||||
# Pick out the data we're interested in
|
||||
status = None
|
||||
for i in range(len(fields)):
|
||||
if fields[i] == 'status':
|
||||
status = values[i]
|
||||
break
|
||||
|
||||
this = res.get(site, Site())
|
||||
this.set_status(group, label, status)
|
||||
res[site] = this
|
||||
|
||||
logger.debug('Parsed status: {}'.format(res))
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def check_site(site, group, status, params):
|
||||
group_backends = status[site].groups.get(group)
|
||||
logger.debug('Processing site {}, group {}, params {}, backends {}'.format(
|
||||
site, group, params, group_backends))
|
||||
backends_up = []
|
||||
if group_backends:
|
||||
backends_up = [x for x in group_backends.keys() if group_backends[x] == 'UP']
|
||||
logger.debug('Backends UP: {}'.format(backends_up))
|
||||
up = len(backends_up)
|
||||
min_up = params.get('min_up', 1)
|
||||
if up < int(min_up):
|
||||
logger.debug('Fewer than {} backends up ({})'.format(min_up, up))
|
||||
if group_backends:
|
||||
no_check = [x for x in group_backends.keys() if group_backends[x] == 'no check']
|
||||
if len(no_check) == len(group_backends):
|
||||
return ['NOCHECK site={}, group={}, backends_up={}'.format(site, group, up)]
|
||||
return ['DOWN site={}, group={}, backends_up={}'.format(site, group, up)]
|
||||
return ['UP site={}, group={}, backends_up={}'.format(site, group, up)]
|
||||
|
||||
|
||||
def main(myname = 'haproxy-status', args = None, logger_in = None, defaults = _defaults):
|
||||
if not args:
|
||||
args = parse_args(defaults)
|
||||
|
||||
global logger
|
||||
# initialize various components
|
||||
if logger_in:
|
||||
logger = logger_in
|
||||
else:
|
||||
logger = logging.getLogger(myname)
|
||||
if args.debug:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
# log to stderr when debugging
|
||||
formatter = logging.Formatter('%(asctime)s %(name)s %(threadName)s: %(levelname)s %(message)s')
|
||||
stream_h = logging.StreamHandler(sys.stderr)
|
||||
stream_h.setFormatter(formatter)
|
||||
logger.addHandler(stream_h)
|
||||
if args.syslog:
|
||||
syslog_h = SysLogHandler(address='/dev/log')
|
||||
formatter = logging.Formatter('%(name)s: %(levelname)s %(message)s')
|
||||
syslog_h.setFormatter(formatter)
|
||||
logger.addHandler(syslog_h)
|
||||
|
||||
|
||||
try:
|
||||
status = get_status(args)
|
||||
except HAProxyStatusError as exc:
|
||||
logger.error(exc)
|
||||
return False
|
||||
|
||||
if not status:
|
||||
return False
|
||||
|
||||
output = []
|
||||
if args.site[0].lower() == 'all':
|
||||
args.site = []
|
||||
for k,v in status.items():
|
||||
for group in v.groups.keys():
|
||||
args.site += ['site={}; group={}'.format(k, group)]
|
||||
for this in sorted(args.site):
|
||||
params = {}
|
||||
if '=' in this:
|
||||
# Parse strings such as 'site=www.dev.eduid.se; group=testing'
|
||||
for kv in this.split(';'):
|
||||
k, v = kv.split('=')
|
||||
k = k.strip()
|
||||
v = v.strip()
|
||||
params[k] = v
|
||||
else:
|
||||
params = {'site': this}
|
||||
logger.debug('Parsed params {}'.format(params))
|
||||
site = params['site']
|
||||
group = 'default' if not 'group' in params else params['group']
|
||||
if site not in status:
|
||||
logger.debug('Site {} not found in haproxy status'.format(site))
|
||||
continue
|
||||
res = check_site(site, group, status, params)
|
||||
if res:
|
||||
output += res
|
||||
|
||||
print('\n'.join(output))
|
||||
return output != []
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
progname = os.path.basename(sys.argv[0])
|
||||
res = main(progname)
|
||||
if res is True:
|
||||
sys.exit(0)
|
||||
if res is False:
|
||||
sys.exit(1)
|
||||
sys.exit(int(res))
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(0)
|
||||
79
fe-common/overlay/opt/frontend/scripts/monitor-haproxy
Executable file
79
fe-common/overlay/opt/frontend/scripts/monitor-haproxy
Executable file
|
|
@ -0,0 +1,79 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Check the status of the haproxy backends every ${INTERVAL} seconds.
|
||||
#
|
||||
# Every time a change in status is detected, frontend-config is used to generate
|
||||
# a new exabgp-announce file for this frontend instance.
|
||||
#
|
||||
# The exabgp monitor script will notice this updated status immediately (by watching
|
||||
# for file change events using inotify) and update it's announcements of this instances
|
||||
# frontend IP addresses.
|
||||
#
|
||||
|
||||
if [[ ! $HOSTFQDN ]]; then
|
||||
echo "$0: ERROR: Environment variable HOSTFQDN not provided"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! $INSTANCE ]]; then
|
||||
echo "$0: ERROR: Environment variable INSTANCE not provided"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INTERVAL=${INTERVAL-'10'}
|
||||
STATUSFN=${STATUSFN-'/var/run/haproxy-status'}
|
||||
OUTFILE=${OUTFILE-"/opt/frontend/monitor/${INSTANCE}/announce"}
|
||||
STATSSOCKET=${STATSSOCKET-'/var/run/haproxy-control/stats'}
|
||||
|
||||
for retry in $(seq 20); do
|
||||
if [ -S ${STATSSOCKET} ]; then
|
||||
/opt/frontend/scripts/haproxy-status $* > ${STATUSFN}
|
||||
grep -qe ^UP -e ^DOWN ${STATUSFN} && break
|
||||
fi
|
||||
echo "$0: haproxy status socket ${STATSSOCKET} not found (attempt ${retry}/20)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
test -S ${STATSSOCKET} || {
|
||||
echo "$0: Could not find haproxy status socket ${STATSSOCKET} - is the haproxy container not running?"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "$0: Startup status is `cat ${STATUSFN}`"
|
||||
|
||||
status=$(cat ${STATUSFN} | awk '{print $1}')
|
||||
/opt/frontend/scripts/frontend-config --debug --fqdn ${HOSTFQDN} --status ${status} --instance ${INSTANCE} print_exabgp_announce > ${OUTFILE}.new
|
||||
mv ${OUTFILE}.new ${OUTFILE}
|
||||
|
||||
pid=0
|
||||
|
||||
term_handler() {
|
||||
echo "$0: Received SIGTERM, shutting down ${pid}"
|
||||
if [ $pid -ne 0 ]; then
|
||||
kill -SIGTERM "$pid"
|
||||
wait "$pid"
|
||||
fi
|
||||
exit 143; # 128 + 15 -- SIGTERM
|
||||
}
|
||||
|
||||
trap 'kill ${!}; term_handler' SIGTERM
|
||||
|
||||
while [ 1 ]; do
|
||||
/opt/frontend/scripts/haproxy-status $* > ${STATUSFN}.new
|
||||
changed=0
|
||||
cmp --quiet "${STATUSFN}.new" "${STATUSFN}" || changed=1
|
||||
if [[ $changed == 1 ]]; then
|
||||
mv ${STATUSFN}.new ${STATUSFN}
|
||||
echo "$0: Status changed to `cat ${STATUSFN}`"
|
||||
|
||||
status=$(cat ${STATUSFN} | awk '{print $1}')
|
||||
/opt/frontend/scripts/frontend-config --debug --fqdn ${HOSTFQDN} --status ${status} --instance ${INSTANCE} print_exabgp_announce > ${OUTFILE}.new
|
||||
mv ${OUTFILE}.new ${OUTFILE}
|
||||
cat ${OUTFILE}
|
||||
fi
|
||||
|
||||
sleep ${INTERVAL} &
|
||||
pid=$!
|
||||
wait ${pid}
|
||||
|
||||
sleep 1 # spin control
|
||||
done
|
||||
|
|
@ -1,4 +1,7 @@
|
|||
---
|
||||
eid_docker_version: '18.02.0~ce-0~ubuntu'
|
||||
eid_docker_compose_version: '1.15.0'
|
||||
|
||||
eid_proxy_server: ""
|
||||
eid_no_proxy: true
|
||||
syslog_servers:
|
||||
|
|
|
|||
|
|
@ -131,3 +131,7 @@ md-eu1.qa.komreg.net:
|
|||
servicemonitor:
|
||||
prid:
|
||||
version: 1.0.1
|
||||
|
||||
'^fe-.+-\d+\.komreg\.net$':
|
||||
eid::dockerhost:
|
||||
sunet::frontend::load_balancer:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
# Wrapper for sunet::dockerhost to do eduID specific things
|
||||
class eid::dockerhost(
|
||||
String $version = safe_hiera('eid_docker_version'),
|
||||
String $package_name = hiera('eid_docker_package_name', 'docker-ce'),
|
||||
Enum['stable', 'edge'] $docker_repo = hiera('eid_docker_repo', 'stable'),
|
||||
String $compose_version = safe_hiera('eid_docker_compose_version'),
|
||||
String $docker_args = '',
|
||||
Optional[String] $docker_dns = undef,
|
||||
) {
|
||||
if $version == 'NOT_SET_IN_HIERA' {
|
||||
fail('Docker version not set in Hiera')
|
||||
}
|
||||
if $compose_version == 'NOT_SET_IN_HIERA' {
|
||||
fail('Docker-compose version not set in Hiera')
|
||||
}
|
||||
class { 'sunet::dockerhost':
|
||||
docker_version => $version,
|
||||
docker_package_name => $package_name,
|
||||
docker_repo => $docker_repo,
|
||||
run_docker_cleanup => true,
|
||||
manage_dockerhost_unbound => true,
|
||||
docker_extra_parameters => $docker_args,
|
||||
docker_dns => $docker_dns,
|
||||
storage_driver => 'aufs',
|
||||
docker_network => true, # let docker choose a network for the 'docker' bridge
|
||||
compose_version => $compose_version,
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue