9
0
Fork 1

Trying to install fody.

This commit is contained in:
Johan Björklund 2024-11-19 10:25:44 +01:00
parent b87dfb39ae
commit 815341ac3e
Signed by: bjorklund
GPG key ID: 5E8401339C7F5037
10 changed files with 361 additions and 18 deletions

View file

@ -0,0 +1,232 @@
-- Initialize the notifications part of the event DB.
--
-- The notifications part keeps track of which emails are to be sent and
-- which have already been sent in the notifications table. There's also
-- a trigger on the events table that automatically extracts the
-- notification information added to events by the certbund_contact bot
-- and inserts it into notifications.
BEGIN;
CREATE ROLE eventdb_owner
NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB CREATEROLE;
CREATE ROLE eventdb_insert
NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB CREATEROLE;
CREATE ROLE eventdb_send_notifications
NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB CREATEROLE;
CREATE SEQUENCE intelmq_ticket_seq MINVALUE 10000001;
GRANT INSERT ON events TO eventdb_insert;
GRANT USAGE ON events_id_seq TO eventdb_insert;
GRANT SELECT ON events TO eventdb_send_notifications;
CREATE TYPE ip_endpoint AS ENUM ('source', 'destination');
-- a single row table to save which day we currently use for intelmq_ticket
CREATE TABLE ticket_day (
initialized_for_day DATE
);
INSERT INTO ticket_day (initialized_for_day) VALUES('20160101');
GRANT SELECT, UPDATE ON ticket_day TO eventdb_send_notifications;
CREATE TABLE sent (
id BIGSERIAL UNIQUE PRIMARY KEY,
intelmq_ticket VARCHAR(18) UNIQUE NOT NULL,
sent_at TIMESTAMP WITH TIME ZONE
);
GRANT SELECT, INSERT ON sent TO eventdb_send_notifications;
GRANT USAGE ON sent_id_seq TO eventdb_send_notifications;
CREATE TABLE directives (
id BIGSERIAL UNIQUE PRIMARY KEY,
events_id BIGINT NOT NULL,
sent_id BIGINT,
medium VARCHAR(100) NOT NULL,
recipient_address VARCHAR(100) NOT NULL,
template_name VARCHAR(100) NOT NULL,
notification_format VARCHAR(100) NOT NULL,
event_data_format VARCHAR(100) NOT NULL,
aggregate_identifier TEXT[][],
notification_interval INTERVAL NOT NULL,
endpoint ip_endpoint NOT NULL,
inserted_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (events_id) REFERENCES events(id),
FOREIGN KEY (sent_id) REFERENCES sent(id)
);
CREATE INDEX directives_grouping_inserted_at_idx
ON directives (recipient_address, template_name,
notification_format, event_data_format,
aggregate_identifier, inserted_at);
CREATE INDEX directives_events_id_idx
ON directives (events_id);
CREATE INDEX directives_sent_id_idx
ON directives (sent_id);
-- Use https://www.postgresql.org/docs/9.5/pgtrgm.html to allow for
-- fast ILIKE search in tags saved in the aggregate_identifier.
-- If additional tags are entered there, additional indixes may be advisable.
CREATE EXTENSION pg_trgm;
CREATE INDEX directives_recipient_group_idx
ON directives USING gist (
(json_object(aggregate_identifier) ->> 'recipient_group')
gist_trgm_ops
);
GRANT SELECT, UPDATE ON directives TO eventdb_send_notifications;
-- Converts a JSON object used as aggregate identifier to a
-- 2-dimensional TEXT array usable as a value in the database for
-- grouping. Doing this properly is a bit tricky. Requirements:
--
-- 1. the type must allow comparison because we need to be able to
-- GROUP BY the aggregate_identifier column
--
-- 2. The value must be chosen to preserve the equivalence relation on
-- the abstract aggregate identifier, meaning
--
-- (a) Equal aggregate identifiers have to be mapped to the equal
-- values
--
-- (b) equal values must imply equal aggregate identifiers
--
-- Requirement 1 rules out using JSON directly because it doesn't
-- support comparison. We cannot use JSONB either because that type is
-- not available in PostgreSQL 9.3 (JSONB requires at least 9.4). Simply
-- converting the JSON object to TEXT is not an option either since, for
-- instance, the order of the keys would not be predictable.
--
-- Requirement 2 means we need to be careful when choosing the
-- representation. An easy solution would be to iterate over the JSON
-- object with the json_each or json_each_text functions. Neither is
-- really good. json_each returns the values as JSON objects in which
-- case the conversion to TEXT will not preserve equality in the case of
-- Strings because escape sequences will not be normalized.
-- json_each_text returns the values as text which means that numbers
-- and strings cannot be distinguished reliably (123 and "123" would be
-- considered equal).
--
-- Given that we might switch to PostgreSQL 9.5 which comes with Ubuntu
-- 16.4 LTS we go with json_each_text because in most cases the values
-- will have come from IntelMQ events where the values have been
-- validated and e.g. ASNs will always be numbers.
CREATE OR REPLACE FUNCTION json_object_as_text_array(obj JSON)
RETURNS TEXT[][]
AS $$
DECLARE
arr TEXT[][] = '{}'::TEXT[][];
k TEXT;
v TEXT;
BEGIN
FOR k, v IN
SELECT * FROM json_each_text(obj) ORDER BY key
LOOP
arr := arr || ARRAY[ARRAY[k, v]];
END LOOP;
RETURN arr;
END
$$ LANGUAGE plpgsql IMMUTABLE;
CREATE OR REPLACE FUNCTION insert_directive(
event_id BIGINT,
directive JSON,
endpoint ip_endpoint
) RETURNS VOID
AS $$
DECLARE
medium TEXT := directive ->> 'medium';
recipient_address TEXT := directive ->> 'recipient_address';
template_name TEXT := directive ->> 'template_name';
notification_format TEXT := directive ->> 'notification_format';
event_data_format TEXT := directive ->> 'event_data_format';
aggregate_identifier TEXT[][]
:= json_object_as_text_array(directive -> 'aggregate_identifier');
notification_interval interval
:= coalesce(((directive ->> 'notification_interval') :: INT)
* interval '1 second',
interval '0 second');
BEGIN
IF medium IS NOT NULL
AND recipient_address IS NOT NULL
AND template_name IS NOT NULL
AND notification_format IS NOT NULL
AND event_data_format IS NOT NULL
AND notification_interval IS NOT NULL
AND notification_interval != interval '-1 second'
THEN
INSERT INTO directives (events_id,
medium,
recipient_address,
template_name,
notification_format,
event_data_format,
aggregate_identifier,
notification_interval,
endpoint)
VALUES (event_id,
medium,
recipient_address,
template_name,
notification_format,
event_data_format,
aggregate_identifier,
notification_interval,
endpoint);
END IF;
END
$$ LANGUAGE plpgsql VOLATILE;
CREATE OR REPLACE FUNCTION directives_from_extra(
event_id BIGINT,
extra JSON
) RETURNS VOID
AS $$
DECLARE
json_directives JSON := extra -> 'certbund' -> 'source_directives';
directive JSON;
BEGIN
IF json_directives IS NOT NULL THEN
FOR directive
IN SELECT * FROM json_array_elements(json_directives) LOOP
PERFORM insert_directive(event_id, directive, 'source');
END LOOP;
END IF;
END
$$ LANGUAGE plpgsql VOLATILE;
CREATE OR REPLACE FUNCTION events_insert_directives_for_row()
RETURNS TRIGGER
AS $$
BEGIN
PERFORM directives_from_extra(NEW.id, NEW.extra);
RETURN NEW;
END
$$ LANGUAGE plpgsql VOLATILE EXTERNAL SECURITY DEFINER;
GRANT EXECUTE ON FUNCTION events_insert_directives_for_row()
TO eventdb_insert;
CREATE TRIGGER events_insert_directive_trigger
AFTER INSERT ON events
FOR EACH ROW
EXECUTE PROCEDURE events_insert_directives_for_row();
COMMIT;

View file

@ -0,0 +1,4 @@
{
"session_store": "/etc/intelmq/fody/fody-session.sqlite",
"session_duration": 86400
}

View file

@ -29,10 +29,12 @@ git clone https://github.com/Intevation/intelmq-certbund-contact.git
git clone https://github.com/Intevation/intelmq-fody-backend.git
git clone https://github.com/Intevation/intelmq-fody.git
( cd /opt/intelmq/src/intelmq-certbund-contact ; pip3 install . ; true )
cp /opt/intelmq/src/intelmq-certbund-contact/sql/initdb.sql /opt/intelmq/install/contactdb-initdb.sql
( cd /opt/intelmq/src/intelmq-fody-backend ; pip3 install . ; true )
for api in tickets_api checkticket_api session contactdb_api events_api; do
(cd /opt/intelmq/src/intelmq-fody-backend/$api ; pip3 install . ; true )
done
sqlite3 /etc/intelmq/fody/fody-session.sqlite < /opt/intelmq/src/intelmq-fody-backend/config/fody-session.sql
mkdir /opt/intelmq/www/fody
( cd /opt/intelmq/src/intelmq-fody ; yarn build ; cd dist ; cp -r * /opt/intelmq/www/fody )
cd

View file

@ -11,9 +11,16 @@ GRANT ALL PRIVILEGES ON DATABASE eventdb TO intelmq;
\c eventdb;
CREATE SCHEMA intelmq AUTHORIZATION intelmq;
EOSQL
sudo -u postgres psql <<-EOSQL
CREATE DATABASE contactdb WITH OWNER intelmq ENCODING UTF8;
GRANT ALL PRIVILEGES ON DATABASE contactdb TO intelmq;
\c contactdb;
CREATE SCHEMA intelmq AUTHORIZATION intelmq;
EOSQL
rm -f /tmp/initdb.sql
sudo -u intelmq /opt/intelmq/venv/bin/intelmq_psql_initdb
sudo -u intelmq psql eventdb -f /tmp/initdb.sql
rm -f /tmp/initdb.sql
touch /opt/intelmq/.evendb-installed
touch /opt/intelmq/.pgsql-installed

View file

@ -7,6 +7,12 @@ class soc::intelmq(
) {
include sunet::systemd_reload
# Set some global variables
$api_user = lookup('intelmq_api_user.username', undef, undef, 'test')
$api_pass = lookup('intelmq_api_user.password', undef, undef, 'pass')
$db_user = lookup('intelmq_db_user.username', undef, undef, 'test')
$db_pass = lookup('intelmq_db_user.password', undef, undef, 'pass')
group { 'intelmq':
ensure => present,
}
@ -33,10 +39,12 @@ class soc::intelmq(
mode => '0770',
}
# file { '/opt/sso/apache/groups.txt':
# ensure => file,
# content => template('soc/sso/apache-groups.txt.erb')
# }
file { '/etc/intelmq/fody':
ensure => directory,
owner => 'intelmq',
group => 'www-data',
mode => '0770',
}
package { 'apache2':
ensure => 'latest',
@ -46,14 +54,21 @@ class soc::intelmq(
ensure => 'latest',
}
file { '/opt/intelmq/setup-nodesource.sh':
file { '/opt/intelmq/install':
ensure => directory,
owner => 'intelmq',
group => 'intelmq',
mode => '0755',
}
file { '/opt/intelmq/install/setup-nodesource.sh':
ensure => file,
content => file('soc/intelmq/setup-nodesource.sh'),
mode => '0500',
mode => '0540',
}
exec { 'Add nodesource repo':
command => '/opt/intelmq/setup-nodesource.sh',
command => '/opt/intelmq/install/setup-nodesource.sh',
creates => '/etc/apt/sources.list.d/nodesource.list',
}
@ -61,7 +76,6 @@ class soc::intelmq(
ensure => 'latest',
}
exec { 'Install IntelMQ venv':
command => 'sudo -u intelmq /usr/bin/python3 -m venv --system-site-packages /opt/intelmq/venv',
creates => '/opt/intelmq/venv',
@ -72,14 +86,19 @@ class soc::intelmq(
unless => 'grep -q activate /opt/intelmq/.profile 2> /dev/null',
}
file { '/opt/intelmq/install-intelmq.sh':
file { '/opt/intelmq/install/eventdb-notifications.sql':
ensure => file,
content => file('soc/intelmq/install-intelmq.sh'),
content => template('soc/intelmq/eventdb-notifications.sql'),
}
file { '/opt/intelmq/install/install-intelmq.sh':
ensure => file,
content => template('soc/intelmq/install-intelmq.sh.erb'),
mode => '0555',
}
exec { 'Install IntelMQ':
command => 'sudo -u intelmq /opt/intelmq/install-intelmq.sh',
command => 'sudo -u intelmq /opt/intelmq/install/install-intelmq.sh',
creates => '/opt/intelmq/.installed'
}
@ -89,15 +108,15 @@ class soc::intelmq(
returns => ['0', '1',],
}
file { '/opt/intelmq/setup-pgsql.sh':
file { '/opt/intelmq/install/setup-pgsql.sh':
ensure => file,
content => file('soc/intelmq/setup-pgsql.sh'),
content => file('soc/intelmq/install/setup-pgsql.sh'),
mode => '0500',
}
exec { 'Setup IntelMQ eventdb':
command => '/opt/intelmq/setup-pgsql.sh',
creates => '/opt/intelmq/.evendb-installed',
command => '/opt/intelmq/install/setup-pgsql.sh',
creates => '/opt/intelmq/.pgsql-installed',
}
file { '/etc/sudoers.d/01_intelmq-api':
@ -143,8 +162,14 @@ class soc::intelmq(
content => file('soc/intelmq/api-config.json'),
}
$api_user = lookup('intelmq_api_user.username', undef, undef, 'test')
$api_pass = lookup('intelmq_api_user.password', undef, undef, 'pass')
file { '/etc/intelmq/fody-session.config':
ensure => file,
owner => 'intelmq',
group => 'intelmq',
mode => '0444',
content => file('soc/intelmq/fody-sesson.config'),
}
exec { 'Setup intelmq-api user':
command => "sudo -u intelmq /opt/intelmq/venv/bin/intelmq-api-adduser --user ${api_user} --password ${api_pass}",
creates => '/etc/intelmq/api/api-session.sqlite',
@ -158,6 +183,35 @@ class soc::intelmq(
mode => '0660'
}
exec { 'Setup fody-api user':
command => "sudo -u intelmq /opt/intelmq/venv/bin/fody-adduser --user ${api_user} --password ${api_pass}",
unless => "sqlite3 /etc/intelmq/api/api-session.sqlite \"SELECT username FROM user WHERE username ='${api_user}'\" | grep -q ${api_user}",
}
file {
'/etc/intelmq/contactdb-serve.conf':
ensure => file,
owner => 'intelmq',
group => 'www-data',
mode => '0440',
content => tempate('soc/intelmq/contactdb-serve.conf.erb'),
;
'/etc/intelmq/eventdb-serve.conf':
ensure => file,
owner => 'intelmq',
group => 'www-data',
mode => '0440',
content => tempate('soc/intelmq/eventdb-serve.conf.erb'),
;
'/etc/intelmq/tickets-serve.conf':
ensure => file,
owner => 'intelmq',
group => 'www-data',
mode => '0440',
content => tempate('soc/intelmq/tickets-serve.conf.erb'),
;
}
file { '/etc/systemd/system/intelmq-api.service':
ensure => file,
content => file('soc/intelmq/intelmq-api.service'),

View file

@ -0,0 +1,4 @@
{
"libpg conninfo": "host=localhost dbname=contactdb user=<%- @db_user %> password=<%- @db_pass %>",
"logging_level": "DEBUG"
}

View file

@ -0,0 +1,5 @@
{
"libpg conninfo": "host=localhost dbname=eventdb user=<%- @db_user %> password=<%- @db_pass %>",
"logging_level": "DEBUG"
}

View file

@ -79,3 +79,29 @@
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
</VirtualHost>
<VirtualHost 127.0.0.1:8666>
ServerAdmin cert@cert.sunet.se
DocumentRoot /opt/intelmq/venv/lib/python3.11/site-packages/intelmq_fody_backend
WSGIDaemonProcess www-fody threads=1 maximum-requests=10000 python-home=/opt/intelmq/venv python-path=/opt/intelmq/venv/lib/python3.11/site-packages
WSGIScriptAlias / /opt/intelmq/venv/lib/python3.11/site-packages/intelmq_fody_backend/serve.py
WSGICallableObject __hug_wsgi__
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
<Directory /opt/intelmq/venv/lib/python3.11/site-packages/intelmq_fody_backend>
<IfModule mod_headers.c>
Header set Content-Security-Policy "script-src 'self'"
Header set X-Content-Security-Policy "script-src 'self'"
</IfModule>
Require all granted
Options FollowSymLinks
</Directory>
LogLevel debug
ErrorLog ${APACHE_LOG_DIR}/fody-backend-error.log
#CustomLog ${APACHE_LOG_DIR}/fody-backend-access.log combined
# combined + logging the time taken to serve the request in microseconds
CustomLog ${APACHE_LOG_DIR}/fody-backend-access.log "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %{ms}T"
</VirtualHost>

View file

@ -0,0 +1,5 @@
{
"libpg conninfo": "host=localhost dbname=eventdb user=<%- @db_user %> password=<%- @db_pass %>",
"logging_level": "DEBUG"
}

View file

@ -2,3 +2,7 @@
intelmq_api_user:
username: sunetcert
password: ENC[PKCS7,MIIC3QYJKoZIhvcNAQcDoIICzjCCAsoCAQAxggKFMIICgQIBADBpMFExCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxIjAgBgNVBAMMGWludGVsbXEtZGV2LmNlcnQuc3VuZXQuc2UCFEo7Xtor0ZsqERHhYlQDoRJ9XhlCMA0GCSqGSIb3DQEBAQUABIICAG+H3gaIqCFrn5oUpslrEjtVza6tPI+q1DHrW6WOLEe/9h6PCeHCEnLdUpaCcFfhTPq7Ds8HLgUhfyK8/upatmGs33UUabh1+RBw0tl+gymuNjz9lD3eIRtT7c2ZEjornNERoeS1EeNcyDv5/YBUdgzyuqnE9CkglvmqQsKgjpQXaptLDFKNg0C6okP+fMLnS8IS+pw4Sk7JBNQTb/Qsg0PT+F/ZFG+s/kT1ceVausNwsJd2xjjg4rbkeJjbWl+8GZY/iTf/gOc18eewrXRcVBcKBDoU/wkiQrAifGork1863e+8gTdlfcypxKqmKpPqUkuZAK1H6l1NJuv94NNp6BGm1j/31k32m1cxqvF6q98n+9yeDqcouz6WDQTn1nSklf9perwHnx+FHFEAYg85A+RZ6l1PQG2ZJ0/gGIzeWraOso0jVayCjNBNRMK3YixzLELFuxBopwlUljBnndcSSvDjc4PgAhC5fWuX1Jt6Xnm9KkpAkQ6uiAyce/RbJoct9Ro7NKMO/4xS/jmUABiqZ8AbrJDxhUj0ctu70jHGNmr69PHL3HVkMSZUL+RRUz3caEdWMdHrNQkuf1cM8fw9JRKgPzl0t2H/Sd73fXmZGSLulXIGz108OWDggSqFRvo8Qbwogv+Y8XtvafOJZAsP8b3c9Fdcsu6yLIym410pXcvKMDwGCSqGSIb3DQEHATAdBglghkgBZQMEASoEEBgs4ZBVnfrtwxxDe10O8iGAEFIMFK/cZZ4H0H5Kxp/DCVE=]
intelmq_db_user:
username: intelmq
password: ENC[PKCS7,MIIC3QYJKoZIhvcNAQcDoIICzjCCAsoCAQAxggKFMIICgQIBADBpMFExCzAJBgNVBAYTAlNFMQ4wDAYDVQQKDAVTVU5FVDEOMAwGA1UECwwFRVlBTUwxIjAgBgNVBAMMGWludGVsbXEtZGV2LmNlcnQuc3VuZXQuc2UCFEo7Xtor0ZsqERHhYlQDoRJ9XhlCMA0GCSqGSIb3DQEBAQUABIICAGaKzrxDtBtK61J+ARmsBBwI9hqrcC5ExeuY+y9xv9h+FYDDHZZIzSrFCJbCCecLby6cgpnlh9kqhU1sYZCiiZFz8Q5aJoCan6bMU+hZi2Yt+hXfHrk+VezlN+jz09BzOdlWY6ETzNnZNx+fukBLEQkMtoCUYZohmDdzpZjx/aFO5nmGwDVi9ks7WX8nBbbm+t5ehGl0fmsOu41AeQf2N93yYGFhKYmjnnc97GHhPzg7E53y/lHlZ9AOqNgmWCLMnGRqxifDcTqQOBvZYJ5TYvPwOmRDLC7a7NIfZ7fRGy6rU5QQDsbImOI+gzWaDlqCstC7EP9R6d+NkLIniRQ5X5DW7e1xlrCxsWSjykjU+qxmsIdJGHwQUGdszvdif9iSsPKrXPF/QyNnYXOcwKNqvHWS6dh3vmym4ZrSt/mz0gsnDbPF9QY4j0eU9s2f+KFw5SeOkIj7/DgfEd7c0rvQtrK4L+DjzFHEurJeQ2liwLicy+jN6EeIw4W5e5hKvn30xLcth85cRh1aCuJl5spLjEXqrGKNy1lJInOXbIDGUEm5SXcpsgZRsMJtsHL0ZPbUc4gZ2LtboM2dNY7EGvAQB6WnA0kZ4QTeRKpoVvh2Q3zoZRtSjI8PXV+ze1KjzrlKP4pgf8pw3Z+tMChuts9JOOBgYzroElkZBXAMN3CsYBwnMDwGCSqGSIb3DQEHATAdBglghkgBZQMEASoEEIqDfpyuW9c7tj7gFKIXli6AEAnQHES6GLDxY67J1mzJPTo=]