SUNET/sysadmin-tools
12
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add support for tsocks

Browse source
This commit is contained in:
Micke Nordin 2024-11-06 15:43:01 +01:00
parent c674a02a1c
commit df681e23dc
Signed by: Micke
GPG key ID: 0DA0A7A5708FE257
1 changed files with 247 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

435
scripts/openstack-server
View file

@ -19,7 +19,7 @@ import argcomplete
# To determine if openstack thinks that a given object exists already
def exists_in_openstack(output_error: tuple) -> bool:
output = output_error[0]
#error = output_error[1]
# error = output_error[1]
# FIXME: This is a dumb way to check that an object exists, we should check the error in addition to this
try:
json.loads(output)
@ -29,12 +29,11 @@ def exists_in_openstack(output_error: tuple) -> bool:
def get_dns_records(addresses: dict, fqdn: str) -> str:
def get_tabs(string: str) -> str:
num_tabs: int = int(len(string.expandtabs()) / len(f"\t".expandtabs()))
tabs = str()
for _ in range(0, num_tabs):
tabs += '\t'
tabs += "\t"
return tabs
v6_record = str()
@ -42,7 +41,7 @@ def get_dns_records(addresses: dict, fqdn: str) -> str:
first = True
tabs = get_tabs(f"{fqdn}.\t")
for address in addresses:
if address.find('.') != -1:
if address.find(".") != -1:
if first:
v4_record = f"{fqdn}.\tA\t{address}\n"
first = False
@ -57,7 +56,7 @@ def get_dns_records(addresses: dict, fqdn: str) -> str:
# This function is responsible for collecting all neccessary information from the user, and provide feedback if something is missing
def get_options() -> dict:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(title='subcommands', dest='command')
subparsers = parser.add_subparsers(title="subcommands", dest="command")
# Used with the command "create"
create_parser = subparsers.add_parser("create")
@ -83,16 +82,14 @@ def get_options() -> dict:
)
create_parser.add_argument(
"--sgroup-policy",
dest='sgroup_policy',
dest="sgroup_policy",
help="Set security group policy for machine",
choices=[
'affinity', 'anti-affinity', 'soft-affinity', 'soft-anti-affinity'
],
choices=["affinity", "anti-affinity", "soft-affinity", "soft-anti-affinity"],
default="anti-affinity",
)
create_parser.add_argument(
"--volume-size",
dest='volume_size',
dest="volume_size",
help="Set volume size in GB for machine",
default="100",
)
@ -100,24 +97,29 @@ def get_options() -> dict:
create_parser.add_argument(
"-k",
"--key",
help=
"SSH key to use, either the path to a public ssh key OR the name of an existing key in openstack",
help="SSH key to use, either the path to a public ssh key OR the name of an existing key in openstack",
required=True,
)
create_parser.add_argument(
"--fqdn",
help="fqdn of server, MUST follow name standard",
required=True)
"--fqdn", help="fqdn of server, MUST follow name standard", required=True
)
create_parser.add_argument(
'--format',
choices=['bind', 'json', 'shell', 'table', 'value', 'yaml'],
"--format",
choices=["bind", "json", "shell", "table", "value", "yaml"],
default="json",
)
create_parser.add_argument(
"--namestandard",
help="Name of name standard",
default='default',
dest='namestandard',
default="default",
dest="namestandard",
)
create_parser.add_argument(
"--with-tsocks",
help="Run with tsocks",
action=argparse.BooleanOptionalAction,
default=False,
dest="with_tsocks",
)
delete_parser = subparsers.add_parser("delete")
@ -129,36 +131,43 @@ def get_options() -> dict:
delete_parser.add_argument(
"--preserve-port",
action=argparse.BooleanOptionalAction,
dest='preserve_port',
dest="preserve_port",
help="Keep the configured port in openstack",
default=False,
)
delete_parser.add_argument(
"--preserve-cosmos-config",
action=argparse.BooleanOptionalAction,
dest='preserve_cosmos_overlay',
dest="preserve_cosmos_overlay",
help="Keep the overlay for this server around",
default=False,
)
delete_parser.add_argument(
"--preserve-disk",
action=argparse.BooleanOptionalAction,
dest='preserve_disk',
dest="preserve_disk",
help="Keep the configured disk in openstack",
default=False,
)
delete_parser.add_argument(
"--namestandard",
help="Name of name standard",
default='default',
dest='namestandard',
default="default",
dest="namestandard",
)
delete_parser.add_argument(
"--with-tsocks",
help="Run with tsocks",
action=argparse.BooleanOptionalAction,
default=False,
dest="with_tsocks",
)
show_parser = subparsers.add_parser("show")
show_parser.add_argument(
'--format',
choices=['bind', 'json', 'shell', 'table', 'value', 'yaml'],
default='json',
dest='format',
"--format",
choices=["bind", "json", "shell", "table", "value", "yaml"],
default="json",
dest="format",
)
show_parser.add_argument(
"--network",
@ -173,8 +182,15 @@ def get_options() -> dict:
show_parser.add_argument(
"--namestandard",
help="Name of name standard",
default='default',
dest='namestandard',
default="default",
dest="namestandard",
)
show_parser.add_argument(
"--with-tsocks",
help="Run with tsocks",
action=argparse.BooleanOptionalAction,
default=False,
dest="with_tsocks",
)
argcomplete.autocomplete(parser)
@ -182,163 +198,183 @@ def get_options() -> dict:
# This is an opinionated name standard that we follow, and WILL rely on for various automation tasks
fqdn = args.fqdn
domain, environment, function, hostname, instance, location, number, service = parse_fqdn(
fqdn, args.namestandard)
domain, environment, function, hostname, instance, location, number, service = (
parse_fqdn(fqdn, args.namestandard)
)
# We will allways put a server in i server group, you can override the default with the --sgroup switch
if ('sgroup' not in args) or (args.sgroup == ""):
if ("sgroup" not in args) or (args.sgroup == ""):
# if it is not set, we will construct a security group name based on other information
sgroup = fqdn.replace('.', '-') + "-sgroup"
sgroup = fqdn.replace(".", "-").replace(number, "") + "sgroup"
else:
sgroup = args.sgroup
# These options MUST match what is used in main function, or there WILL be dragons
options: dict = {
'command': args.command,
'domain': domain,
'environment': environment,
'fqdn': fqdn,
'function': function,
'hostname': hostname,
'instance': instance,
'location': location,
'namestandard': args.namestandard,
'number': number,
'service': service,
'sgroup': sgroup
"command": args.command,
"domain": domain,
"environment": environment,
"fqdn": fqdn,
"function": function,
"hostname": hostname,
"instance": instance,
"location": location,
"namestandard": args.namestandard,
"number": number,
"service": service,
"sgroup": sgroup,
"with_tsocks": args.with_tsocks,
}
if args.command == 'create':
options['format'] = args.format
options['flavor'] = args.flavor
options['image'] = args.image
options['key'] = args.key
options['network'] = args.network
options['sgroup_policy'] = args.sgroup_policy
options['volume_size'] = args.volume_size
if args.command == 'delete':
options['preserve_port'] = args.preserve_port
options['preserve_cosmos_overlay'] = args.preserve_cosmos_overlay
options['preserve_disk'] = args.preserve_disk
if args.command == 'show':
options['format'] = args.format
if args.command == "create":
options["format"] = args.format
options["flavor"] = args.flavor
options["image"] = args.image
options["key"] = args.key
options["network"] = args.network
options["sgroup_policy"] = args.sgroup_policy
options["volume_size"] = args.volume_size
if args.command == "delete":
options["preserve_port"] = args.preserve_port
options["preserve_cosmos_overlay"] = args.preserve_cosmos_overlay
options["preserve_disk"] = args.preserve_disk
if args.command == "show":
options["format"] = args.format
return options
def parse_fqdn(fqdn: str, namestandard: str) -> tuple:
if namestandard == 'drive':
if namestandard == "drive":
return parse_drive_fqdn(fqdn)
else:
return parse_default_fqdn(fqdn)
def parse_default_fqdn(fqdn: str) -> tuple:
domain = '.'.join([x for x in fqdn.split('.')[:1]])
hostname = fqdn.split('.')[0]
instance, location, environment, function, number = hostname.split('-')
service = fqdn.split('.')[1]
domain = ".".join([x for x in fqdn.split(".")[:1]])
hostname = fqdn.split(".")[0]
instance, location, environment, function, number = hostname.split("-")
service = fqdn.split(".")[1]
return domain, environment, function, hostname, instance, location, number, service
def parse_drive_fqdn(fqdn: str) -> tuple:
type_regex = r'-*[1-9]*\..*'
server_type = re.sub(type_regex, '', fqdn)
type_regex = r"-*[1-9]*\..*"
server_type = re.sub(type_regex, "", fqdn)
env_regex = r'.*(pilot|test).*'
environment = re.sub(env_regex, r'\1', fqdn)
domain = f'drive.{environment}.sunet.se'
customer = 'common'
env_regex = r".*(pilot|test).*"
environment = re.sub(env_regex, r"\1", fqdn)
domain = f"drive.{environment}.sunet.se"
customer = "common"
if environment not in ['pilot', 'test']:
environment = 'prod'
domain = 'drive.sunet.se'
if environment not in ["pilot", "test"]:
environment = "prod"
domain = "drive.sunet.se"
if server_type in ["backup", "intern-db", "node", "redis", "script"]:
customer = fqdn.split('.')[1]
if customer == 'drive':
customer = 'common'
elif server_type in ['gss', 'lookup']:
customer = fqdn.split(".")[1]
if customer == "drive":
customer = "common"
elif server_type in ["gss", "lookup"]:
customer = server_type
elif server_type == 'gssbackup':
customer = 'gss'
elif server_type == 'lookupbackup':
customer = 'lookup'
elif server_type == "gssbackup":
customer = "gss"
elif server_type == "lookupbackup":
customer = "lookup"
else:
customer = 'common'
hostname = fqdn.split('.')[0]
number = re.sub(r'[-a-z.]', '', hostname)
location = 'sto4'
if number == '3':
location = 'sto3'
if number == '1':
location = 'dco'
return domain, environment, server_type, hostname, customer, location, number, 'drive'
customer = "common"
hostname = fqdn.split(".")[0]
number = re.sub(r"[-a-z.]", "", hostname)
location = "sto4"
if number == "3":
location = "sto3"
if number == "1":
location = "dco"
return (
domain,
environment,
server_type,
hostname,
customer,
location,
number,
"drive",
)
def run_in_openstack(outer_command: list[str], rc_file: str) -> tuple:
def run_in_openstack(
outer_command: list[str], rc_file: str, with_tsocks: bool
) -> tuple:
# This function runs the specified command with openstack cli using tsocks.
# FIXME: We should support disabling tsocks with a commandline switch
def run_command(inner_command: list[str]) -> tuple:
proc = subprocess.Popen(inner_command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
proc = subprocess.Popen(
inner_command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
output_error = proc.communicate()
proc.wait()
return output_error
outer_command.insert(0, "openstack")
outer_command.insert(0, 'tsocks')
if with_tsocks:
outer_command.insert(0, "tsocks")
# This is where we actually use the openstack openrc file, and we source it before running the command
string_cmd = f"bash -c 'source {rc_file} &&"
for cmd in outer_command:
string_cmd += ' ' + cmd
string_cmd += " " + cmd
string_cmd += "'"
full_command = shlex.split(string_cmd)
return run_command(full_command)
def setup_key(in_key: str, rc_file: str) -> str:
def setup_key(in_key: str, rc_file: str, with_tsocks: bool) -> str:
if type(in_key) == type(tuple()):
key = in_key[0]
else:
key = in_key
# We deal with public keys first, if the supplied option is a filename we will use that to create a new keypair in openstack if neccessary
# If the key is NOT a file on the local machine, we assume that it is a name of a pre existing key
def keypair_existence(key_name: str) -> bool:
show_keypair_command = ['keypair', 'show', '-f', 'json', key_name]
show_keypair_command = ["keypair", "show", "-f", "json", key_name]
return exists_in_openstack(
run_in_openstack(show_keypair_command, rc_file))
run_in_openstack(show_keypair_command, rc_file, with_tsocks)
)
if isfile(key):
key_name = str(os.environ.get('USER')) + '-sshkey'
key_name = str(os.environ.get("USER")) + "-sshkey"
pub_key = key
create_keypair_command = [
'keypair', 'create', '--public-key', pub_key, key_name
"keypair",
"create",
"--public-key",
pub_key,
key_name,
]
keypair_exists = keypair_existence(key_name)
if not keypair_exists:
run_in_openstack(create_keypair_command, rc_file)
run_in_openstack(create_keypair_command, rc_file, with_tsocks)
else:
key_name = key
keypair_exists = keypair_existence(key_name)
if not keypair_exists:
# We cannot proceed without a public key
print(
'{"ERROR": "No keypair exists and I could not create a new one :("}'
)
print('{"ERROR": "No keypair exists and I could not create a new one :("}')
sys.exit(3)
return key_name
def setup_rc_file(options: dict) -> str:
config_basepath = path_join(path_join(os.environ["HOME"], ".config"),
"sunet")
config_basepath = path_join(path_join(os.environ["HOME"], ".config"), "sunet")
rc_file = path_join(
config_basepath,
f"app-cred-{options['service']}-{options['location']}-{options['environment']}-openrc.sh"
f"app-cred-{options['service']}-{options['location']}-{options['environment']}-openrc.sh",
)
if not isfile(rc_file):
@ -349,21 +385,22 @@ def setup_rc_file(options: dict) -> str:
return rc_file
def show_server(fqdn: str, rc_file: str, format: str = 'json') -> tuple:
show_server_command = ['server', 'show', '-f', format, fqdn]
return run_in_openstack(show_server_command, rc_file)
def show_server(
fqdn: str, rc_file: str, with_tsocks: bool, format: str = "json"
) -> tuple:
show_server_command = ["server", "show", "-f", format, fqdn]
return run_in_openstack(show_server_command, rc_file, with_tsocks)
def wait_for_server(fqdn: str,
rc_file: str,
network: str,
timeout=360) -> bool:
def wait_for_server(
fqdn: str, rc_file: str, network: str, with_tsocks: bool, timeout=360
) -> bool:
then = time.time()
now = then
while then >= (now - timeout):
try:
output, _ = show_server(fqdn, rc_file)
json.loads(output)['addresses'][network]
output, _ = show_server(fqdn, rc_file, with_tsocks)
json.loads(output)["addresses"][network]
return True
except KeyError:
time.sleep(3)
@ -377,128 +414,145 @@ def wait_for_server(fqdn: str,
def create(options: dict) -> int:
# All of these can be set by command line arguments, but some have defaults and as such are optional.
# At this point they MUST be set after parsing args, otherwise there WILL be bugs
key: str = options['key']
key: str = options["key"]
dns_record: bool = False
format: str = options['format']
if format == 'bind':
format = 'json'
format: str = options["format"]
if format == "bind":
format = "json"
dns_record = True
flavor: str = options['flavor']
fqdn: str = options['fqdn']
hostname: str = options['hostname']
image: str = options['image']
network: str = options['network']
rc_file: str = options['rc_file']
sgroup: str = options['sgroup']
sgroup_policy: str = options['sgroup_policy']
volume_size: str = options['volume_size']
key_name: str = setup_key(key, rc_file)
flavor: str = options["flavor"]
fqdn: str = options["fqdn"]
hostname: str = options["hostname"]
image: str = options["image"]
network: str = options["network"]
rc_file: str = options["rc_file"]
sgroup: str = options["sgroup"]
sgroup_policy: str = options["sgroup_policy"]
volume_size: str = options["volume_size"]
with_tsocks: bool = options["with_tsocks"]
key_name: str = setup_key(key, rc_file, with_tsocks)
# FIXME: we don't support setting the portname, we just use the fqdn and append -port to it
port_name = options['fqdn'].replace('.', '-') + '-port'
port_name = options["fqdn"].replace(".", "-") + "-port"
# Internal helper functions
def sgroup_show(sgroup: str) -> tuple:
show_sgroup_command = ['server', 'group', 'show', '-f', 'json', sgroup]
return run_in_openstack(show_sgroup_command, rc_file)
show_sgroup_command = ["server", "group", "show", "-f", "json", sgroup]
return run_in_openstack(show_sgroup_command, rc_file, with_tsocks)
# Check if server exists
server_output_error = show_server(fqdn, rc_file)
server_output_error = show_server(fqdn, rc_file, with_tsocks)
server_exists = exists_in_openstack(server_output_error)
# We will bail out early if the server allready exists, and just print out what we know about it
if server_exists:
if dns_record:
addresses = json.loads(
server_output_error[0])['addresses'][network]
addresses = json.loads(server_output_error[0])["addresses"][network]
print(get_dns_records(addresses, fqdn))
else:
if format == 'json':
if format == "json":
print(server_output_error[0])
else:
print(show_server(fqdn, rc_file, format)[0])
print(show_server(fqdn, rc_file, with_tsocks, format)[0])
return 0
# Server group section
sgroup_exists = exists_in_openstack(sgroup_show(sgroup))
if not sgroup_exists:
create_sgroup_command = [
'server', 'group', 'create', '--policy', sgroup_policy, sgroup
"server",
"group",
"create",
"--policy",
sgroup_policy,
sgroup,
]
run_in_openstack(create_sgroup_command, rc_file)
run_in_openstack(create_sgroup_command, rc_file, with_tsocks)
sgroup_output, _ = sgroup_show(sgroup)
sgroup_object = json.loads(sgroup_output)
sgroup_id = sgroup_object['id']
sgroup_id = sgroup_object["id"]
# Port section
show_port_command = ['port', 'show', '-f', 'json', port_name]
show_port_command = ["port", "show", "-f", "json", port_name]
port_exists = exists_in_openstack(
run_in_openstack(show_port_command, rc_file))
run_in_openstack(show_port_command, rc_file, with_tsocks)
)
if not port_exists:
create_port_command = [
'port', 'create', '--network', network, port_name
]
run_in_openstack(create_port_command, rc_file)
create_port_command = ["port", "create", "--network", network, port_name]
run_in_openstack(create_port_command, rc_file, with_tsocks)
# Finaly set up is done, and we can create server
create_server_command = [
'server', 'create', '--port', port_name, '--flavor', flavor, '--image',
image, '--boot-from-volume', volume_size, '--key-name', key_name,
'--hint', f'group={sgroup_id}', fqdn
"server",
"create",
"--port",
port_name,
"--flavor",
flavor,
"--image",
image,
"--boot-from-volume",
volume_size,
"--key-name",
key_name,
"--hint",
f"group={sgroup_id}",
fqdn,
]
run_in_openstack(create_server_command, rc_file)
run_in_openstack(create_server_command, rc_file, with_tsocks)
timeout = 360
succeed = wait_for_server(fqdn, rc_file, network, timeout=timeout)
succeed = wait_for_server(fqdn, rc_file, network, with_tsocks, timeout=timeout)
if not succeed:
print(f"Wait for server: {fqdn} timed out in {timeout}")
return 4
if dns_record:
output, _ = show_server(fqdn, rc_file)
addresses = json.loads(output)['addresses'][network]
output, _ = show_server(fqdn, rc_file, with_tsocks)
addresses = json.loads(output)["addresses"][network]
print(get_dns_records(addresses, fqdn))
else:
print(show_server(fqdn, rc_file, format))
print(show_server(fqdn, rc_file, with_tsocks, format))
return 0
def delete(options: dict) -> int:
fqdn = options['fqdn']
rc_file = options['rc_file']
preserve_port = options['preserve_port']
preserve_cosmos_overlay = options['preserve_cosmos_overlay']
preserve_disk = options['preserve_disk']
fqdn = options["fqdn"]
rc_file = options["rc_file"]
preserve_port = options["preserve_port"]
preserve_cosmos_overlay = options["preserve_cosmos_overlay"]
preserve_disk = options["preserve_disk"]
with_tsocks = options["with_tsocks"]
# Check if server exists
server_output_error = show_server(fqdn, rc_file)
server_output_error = show_server(fqdn, rc_file, with_tsocks)
server_exists = exists_in_openstack(server_output_error)
# We will bail out early if the server does not allready exists
if not server_exists:
print(f"No server called {fqdn} exists.")
return 0
delete_server_command = ['server', 'delete', fqdn]
run_in_openstack(delete_server_command, rc_file)
delete_server_command = ["server", "delete", fqdn]
run_in_openstack(delete_server_command, rc_file, with_tsocks)
print(f"Deleted server: {fqdn}")
if not preserve_disk:
server_object = json.loads(server_output_error[0])
for disk in server_object['volumes_attached']:
delete_disk_command = ['volume', 'delete', disk['id']]
run_in_openstack(delete_disk_command, rc_file)
for disk in server_object["volumes_attached"]:
delete_disk_command = ["volume", "delete", disk["id"]]
run_in_openstack(delete_disk_command, rc_file, with_tsocks)
# Port section
if not preserve_port:
# FIXME: we don't support setting the portname, we just use the fqdn and append -port to it
port_name = options['fqdn'].replace('.', '-') + '-port'
show_port_command = ['port', 'show', '-f', 'json', port_name]
port_name = options["fqdn"].replace(".", "-") + "-port"
show_port_command = ["port", "show", "-f", "json", port_name]
port_exists = exists_in_openstack(
run_in_openstack(show_port_command, rc_file))
run_in_openstack(show_port_command, rc_file, with_tsocks)
)
if port_exists:
delete_port_command = ['port', 'delete', port_name]
run_in_openstack(delete_port_command, rc_file)
delete_port_command = ["port", "delete", port_name]
run_in_openstack(delete_port_command, rc_file, with_tsocks)
print(f"Deleted port: {port_name}")
dir = os.path.dirname(os.path.abspath(__file__))
overlay = path_join(dir, fqdn)
@ -509,29 +563,34 @@ def delete(options: dict) -> int:
def show(options: dict) -> None:
if options['format'] == 'bind':
if options["format"] == "bind":
# We assume that only public addresses go in dns
addresses = json.loads(
show_server(options['fqdn'],
options['rc_file'])[0])['addresses']['public']
print(get_dns_records(addresses, options['fqdn']))
show_server(options["fqdn"], options["rc_file"], options["with_tsocks"])[0]
)["addresses"]["public"]
print(get_dns_records(addresses, options["fqdn"]))
else:
print(
show_server(options['fqdn'], options['rc_file'],
options['format']))
show_server(
options["fqdn"],
options["rc_file"],
options["with_tsocks"],
options["format"],
)
)
def main() -> int:
# Make sure we have openstack app credentials matching the naming scheme <service>-<location>-<environment>
options = get_options()
options['rc_file'] = setup_rc_file(options)
command = options['command']
if command == 'create':
options["rc_file"] = setup_rc_file(options)
command = options["command"]
if command == "create":
return create(options)
if command == 'delete':
if command == "delete":
return delete(options)
if command == 'show':
if command == "show":
show(options)
return 0