From df681e23dced7d9f8aa4424894a147d3007cc962 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Wed, 6 Nov 2024 15:43:01 +0100 Subject: [PATCH] Add support for tsocks --- scripts/openstack-server | 435 ++++++++++++++++++++++----------------- 1 file changed, 247 insertions(+), 188 deletions(-) diff --git a/scripts/openstack-server b/scripts/openstack-server index 54a387a..3cb5f3f 100755 --- a/scripts/openstack-server +++ b/scripts/openstack-server @@ -19,7 +19,7 @@ import argcomplete # To determine if openstack thinks that a given object exists already def exists_in_openstack(output_error: tuple) -> bool: output = output_error[0] - #error = output_error[1] + # error = output_error[1] # FIXME: This is a dumb way to check that an object exists, we should check the error in addition to this try: json.loads(output) @@ -29,12 +29,11 @@ def exists_in_openstack(output_error: tuple) -> bool: def get_dns_records(addresses: dict, fqdn: str) -> str: - def get_tabs(string: str) -> str: num_tabs: int = int(len(string.expandtabs()) / len(f"\t".expandtabs())) tabs = str() for _ in range(0, num_tabs): - tabs += '\t' + tabs += "\t" return tabs v6_record = str() @@ -42,7 +41,7 @@ def get_dns_records(addresses: dict, fqdn: str) -> str: first = True tabs = get_tabs(f"{fqdn}.\t") for address in addresses: - if address.find('.') != -1: + if address.find(".") != -1: if first: v4_record = f"{fqdn}.\tA\t{address}\n" first = False @@ -57,7 +56,7 @@ def get_dns_records(addresses: dict, fqdn: str) -> str: # This function is responsible for collecting all neccessary information from the user, and provide feedback if something is missing def get_options() -> dict: parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(title='subcommands', dest='command') + subparsers = parser.add_subparsers(title="subcommands", dest="command") # Used with the command "create" create_parser = subparsers.add_parser("create") @@ -83,16 +82,14 @@ def get_options() -> dict: ) create_parser.add_argument( "--sgroup-policy", - dest='sgroup_policy', + dest="sgroup_policy", help="Set security group policy for machine", - choices=[ - 'affinity', 'anti-affinity', 'soft-affinity', 'soft-anti-affinity' - ], + choices=["affinity", "anti-affinity", "soft-affinity", "soft-anti-affinity"], default="anti-affinity", ) create_parser.add_argument( "--volume-size", - dest='volume_size', + dest="volume_size", help="Set volume size in GB for machine", default="100", ) @@ -100,24 +97,29 @@ def get_options() -> dict: create_parser.add_argument( "-k", "--key", - help= - "SSH key to use, either the path to a public ssh key OR the name of an existing key in openstack", + help="SSH key to use, either the path to a public ssh key OR the name of an existing key in openstack", required=True, ) create_parser.add_argument( - "--fqdn", - help="fqdn of server, MUST follow name standard", - required=True) + "--fqdn", help="fqdn of server, MUST follow name standard", required=True + ) create_parser.add_argument( - '--format', - choices=['bind', 'json', 'shell', 'table', 'value', 'yaml'], + "--format", + choices=["bind", "json", "shell", "table", "value", "yaml"], default="json", ) create_parser.add_argument( "--namestandard", help="Name of name standard", - default='default', - dest='namestandard', + default="default", + dest="namestandard", + ) + create_parser.add_argument( + "--with-tsocks", + help="Run with tsocks", + action=argparse.BooleanOptionalAction, + default=False, + dest="with_tsocks", ) delete_parser = subparsers.add_parser("delete") @@ -129,36 +131,43 @@ def get_options() -> dict: delete_parser.add_argument( "--preserve-port", action=argparse.BooleanOptionalAction, - dest='preserve_port', + dest="preserve_port", help="Keep the configured port in openstack", default=False, ) delete_parser.add_argument( "--preserve-cosmos-config", action=argparse.BooleanOptionalAction, - dest='preserve_cosmos_overlay', + dest="preserve_cosmos_overlay", help="Keep the overlay for this server around", default=False, ) delete_parser.add_argument( "--preserve-disk", action=argparse.BooleanOptionalAction, - dest='preserve_disk', + dest="preserve_disk", help="Keep the configured disk in openstack", default=False, ) delete_parser.add_argument( "--namestandard", help="Name of name standard", - default='default', - dest='namestandard', + default="default", + dest="namestandard", + ) + delete_parser.add_argument( + "--with-tsocks", + help="Run with tsocks", + action=argparse.BooleanOptionalAction, + default=False, + dest="with_tsocks", ) show_parser = subparsers.add_parser("show") show_parser.add_argument( - '--format', - choices=['bind', 'json', 'shell', 'table', 'value', 'yaml'], - default='json', - dest='format', + "--format", + choices=["bind", "json", "shell", "table", "value", "yaml"], + default="json", + dest="format", ) show_parser.add_argument( "--network", @@ -173,8 +182,15 @@ def get_options() -> dict: show_parser.add_argument( "--namestandard", help="Name of name standard", - default='default', - dest='namestandard', + default="default", + dest="namestandard", + ) + show_parser.add_argument( + "--with-tsocks", + help="Run with tsocks", + action=argparse.BooleanOptionalAction, + default=False, + dest="with_tsocks", ) argcomplete.autocomplete(parser) @@ -182,163 +198,183 @@ def get_options() -> dict: # This is an opinionated name standard that we follow, and WILL rely on for various automation tasks fqdn = args.fqdn - domain, environment, function, hostname, instance, location, number, service = parse_fqdn( - fqdn, args.namestandard) + domain, environment, function, hostname, instance, location, number, service = ( + parse_fqdn(fqdn, args.namestandard) + ) # We will allways put a server in i server group, you can override the default with the --sgroup switch - if ('sgroup' not in args) or (args.sgroup == ""): + if ("sgroup" not in args) or (args.sgroup == ""): # if it is not set, we will construct a security group name based on other information - sgroup = fqdn.replace('.', '-') + "-sgroup" + sgroup = fqdn.replace(".", "-").replace(number, "") + "sgroup" else: sgroup = args.sgroup # These options MUST match what is used in main function, or there WILL be dragons options: dict = { - 'command': args.command, - 'domain': domain, - 'environment': environment, - 'fqdn': fqdn, - 'function': function, - 'hostname': hostname, - 'instance': instance, - 'location': location, - 'namestandard': args.namestandard, - 'number': number, - 'service': service, - 'sgroup': sgroup + "command": args.command, + "domain": domain, + "environment": environment, + "fqdn": fqdn, + "function": function, + "hostname": hostname, + "instance": instance, + "location": location, + "namestandard": args.namestandard, + "number": number, + "service": service, + "sgroup": sgroup, + "with_tsocks": args.with_tsocks, } - if args.command == 'create': - options['format'] = args.format - options['flavor'] = args.flavor - options['image'] = args.image - options['key'] = args.key - options['network'] = args.network - options['sgroup_policy'] = args.sgroup_policy - options['volume_size'] = args.volume_size - if args.command == 'delete': - options['preserve_port'] = args.preserve_port - options['preserve_cosmos_overlay'] = args.preserve_cosmos_overlay - options['preserve_disk'] = args.preserve_disk - if args.command == 'show': - options['format'] = args.format + if args.command == "create": + options["format"] = args.format + options["flavor"] = args.flavor + options["image"] = args.image + options["key"] = args.key + options["network"] = args.network + options["sgroup_policy"] = args.sgroup_policy + options["volume_size"] = args.volume_size + if args.command == "delete": + options["preserve_port"] = args.preserve_port + options["preserve_cosmos_overlay"] = args.preserve_cosmos_overlay + options["preserve_disk"] = args.preserve_disk + if args.command == "show": + options["format"] = args.format return options + def parse_fqdn(fqdn: str, namestandard: str) -> tuple: - if namestandard == 'drive': + if namestandard == "drive": return parse_drive_fqdn(fqdn) else: return parse_default_fqdn(fqdn) + def parse_default_fqdn(fqdn: str) -> tuple: - domain = '.'.join([x for x in fqdn.split('.')[:1]]) - hostname = fqdn.split('.')[0] - instance, location, environment, function, number = hostname.split('-') - service = fqdn.split('.')[1] + domain = ".".join([x for x in fqdn.split(".")[:1]]) + hostname = fqdn.split(".")[0] + instance, location, environment, function, number = hostname.split("-") + service = fqdn.split(".")[1] return domain, environment, function, hostname, instance, location, number, service def parse_drive_fqdn(fqdn: str) -> tuple: - type_regex = r'-*[1-9]*\..*' - server_type = re.sub(type_regex, '', fqdn) + type_regex = r"-*[1-9]*\..*" + server_type = re.sub(type_regex, "", fqdn) - env_regex = r'.*(pilot|test).*' - environment = re.sub(env_regex, r'\1', fqdn) - domain = f'drive.{environment}.sunet.se' - customer = 'common' + env_regex = r".*(pilot|test).*" + environment = re.sub(env_regex, r"\1", fqdn) + domain = f"drive.{environment}.sunet.se" + customer = "common" - if environment not in ['pilot', 'test']: - environment = 'prod' - domain = 'drive.sunet.se' + if environment not in ["pilot", "test"]: + environment = "prod" + domain = "drive.sunet.se" if server_type in ["backup", "intern-db", "node", "redis", "script"]: - customer = fqdn.split('.')[1] - if customer == 'drive': - customer = 'common' - elif server_type in ['gss', 'lookup']: + customer = fqdn.split(".")[1] + if customer == "drive": + customer = "common" + elif server_type in ["gss", "lookup"]: customer = server_type - elif server_type == 'gssbackup': - customer = 'gss' - elif server_type == 'lookupbackup': - customer = 'lookup' + elif server_type == "gssbackup": + customer = "gss" + elif server_type == "lookupbackup": + customer = "lookup" else: - customer = 'common' - hostname = fqdn.split('.')[0] - number = re.sub(r'[-a-z.]', '', hostname) - location = 'sto4' - if number == '3': - location = 'sto3' - if number == '1': - location = 'dco' - return domain, environment, server_type, hostname, customer, location, number, 'drive' + customer = "common" + hostname = fqdn.split(".")[0] + number = re.sub(r"[-a-z.]", "", hostname) + location = "sto4" + if number == "3": + location = "sto3" + if number == "1": + location = "dco" + return ( + domain, + environment, + server_type, + hostname, + customer, + location, + number, + "drive", + ) -def run_in_openstack(outer_command: list[str], rc_file: str) -> tuple: - +def run_in_openstack( + outer_command: list[str], rc_file: str, with_tsocks: bool +) -> tuple: # This function runs the specified command with openstack cli using tsocks. # FIXME: We should support disabling tsocks with a commandline switch def run_command(inner_command: list[str]) -> tuple: - proc = subprocess.Popen(inner_command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True) + proc = subprocess.Popen( + inner_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) output_error = proc.communicate() proc.wait() return output_error outer_command.insert(0, "openstack") - outer_command.insert(0, 'tsocks') + if with_tsocks: + outer_command.insert(0, "tsocks") # This is where we actually use the openstack openrc file, and we source it before running the command string_cmd = f"bash -c 'source {rc_file} &&" for cmd in outer_command: - string_cmd += ' ' + cmd + string_cmd += " " + cmd string_cmd += "'" full_command = shlex.split(string_cmd) return run_command(full_command) -def setup_key(in_key: str, rc_file: str) -> str: +def setup_key(in_key: str, rc_file: str, with_tsocks: bool) -> str: if type(in_key) == type(tuple()): key = in_key[0] else: key = in_key + # We deal with public keys first, if the supplied option is a filename we will use that to create a new keypair in openstack if neccessary # If the key is NOT a file on the local machine, we assume that it is a name of a pre existing key def keypair_existence(key_name: str) -> bool: - show_keypair_command = ['keypair', 'show', '-f', 'json', key_name] + show_keypair_command = ["keypair", "show", "-f", "json", key_name] return exists_in_openstack( - run_in_openstack(show_keypair_command, rc_file)) + run_in_openstack(show_keypair_command, rc_file, with_tsocks) + ) if isfile(key): - key_name = str(os.environ.get('USER')) + '-sshkey' + key_name = str(os.environ.get("USER")) + "-sshkey" pub_key = key create_keypair_command = [ - 'keypair', 'create', '--public-key', pub_key, key_name + "keypair", + "create", + "--public-key", + pub_key, + key_name, ] keypair_exists = keypair_existence(key_name) if not keypair_exists: - run_in_openstack(create_keypair_command, rc_file) + run_in_openstack(create_keypair_command, rc_file, with_tsocks) else: key_name = key keypair_exists = keypair_existence(key_name) if not keypair_exists: # We cannot proceed without a public key - print( - '{"ERROR": "No keypair exists and I could not create a new one :("}' - ) + print('{"ERROR": "No keypair exists and I could not create a new one :("}') sys.exit(3) return key_name def setup_rc_file(options: dict) -> str: - config_basepath = path_join(path_join(os.environ["HOME"], ".config"), - "sunet") + config_basepath = path_join(path_join(os.environ["HOME"], ".config"), "sunet") rc_file = path_join( config_basepath, - f"app-cred-{options['service']}-{options['location']}-{options['environment']}-openrc.sh" + f"app-cred-{options['service']}-{options['location']}-{options['environment']}-openrc.sh", ) if not isfile(rc_file): @@ -349,21 +385,22 @@ def setup_rc_file(options: dict) -> str: return rc_file -def show_server(fqdn: str, rc_file: str, format: str = 'json') -> tuple: - show_server_command = ['server', 'show', '-f', format, fqdn] - return run_in_openstack(show_server_command, rc_file) +def show_server( + fqdn: str, rc_file: str, with_tsocks: bool, format: str = "json" +) -> tuple: + show_server_command = ["server", "show", "-f", format, fqdn] + return run_in_openstack(show_server_command, rc_file, with_tsocks) -def wait_for_server(fqdn: str, - rc_file: str, - network: str, - timeout=360) -> bool: +def wait_for_server( + fqdn: str, rc_file: str, network: str, with_tsocks: bool, timeout=360 +) -> bool: then = time.time() now = then while then >= (now - timeout): try: - output, _ = show_server(fqdn, rc_file) - json.loads(output)['addresses'][network] + output, _ = show_server(fqdn, rc_file, with_tsocks) + json.loads(output)["addresses"][network] return True except KeyError: time.sleep(3) @@ -377,128 +414,145 @@ def wait_for_server(fqdn: str, def create(options: dict) -> int: # All of these can be set by command line arguments, but some have defaults and as such are optional. # At this point they MUST be set after parsing args, otherwise there WILL be bugs - key: str = options['key'] + key: str = options["key"] dns_record: bool = False - format: str = options['format'] - if format == 'bind': - format = 'json' + format: str = options["format"] + if format == "bind": + format = "json" dns_record = True - flavor: str = options['flavor'] - fqdn: str = options['fqdn'] - hostname: str = options['hostname'] - image: str = options['image'] - network: str = options['network'] - rc_file: str = options['rc_file'] - sgroup: str = options['sgroup'] - sgroup_policy: str = options['sgroup_policy'] - volume_size: str = options['volume_size'] - - key_name: str = setup_key(key, rc_file) + flavor: str = options["flavor"] + fqdn: str = options["fqdn"] + hostname: str = options["hostname"] + image: str = options["image"] + network: str = options["network"] + rc_file: str = options["rc_file"] + sgroup: str = options["sgroup"] + sgroup_policy: str = options["sgroup_policy"] + volume_size: str = options["volume_size"] + with_tsocks: bool = options["with_tsocks"] + key_name: str = setup_key(key, rc_file, with_tsocks) # FIXME: we don't support setting the portname, we just use the fqdn and append -port to it - port_name = options['fqdn'].replace('.', '-') + '-port' + port_name = options["fqdn"].replace(".", "-") + "-port" # Internal helper functions def sgroup_show(sgroup: str) -> tuple: - show_sgroup_command = ['server', 'group', 'show', '-f', 'json', sgroup] - return run_in_openstack(show_sgroup_command, rc_file) + show_sgroup_command = ["server", "group", "show", "-f", "json", sgroup] + return run_in_openstack(show_sgroup_command, rc_file, with_tsocks) # Check if server exists - server_output_error = show_server(fqdn, rc_file) + server_output_error = show_server(fqdn, rc_file, with_tsocks) server_exists = exists_in_openstack(server_output_error) # We will bail out early if the server allready exists, and just print out what we know about it if server_exists: if dns_record: - addresses = json.loads( - server_output_error[0])['addresses'][network] + addresses = json.loads(server_output_error[0])["addresses"][network] print(get_dns_records(addresses, fqdn)) else: - if format == 'json': + if format == "json": print(server_output_error[0]) else: - print(show_server(fqdn, rc_file, format)[0]) + print(show_server(fqdn, rc_file, with_tsocks, format)[0]) return 0 # Server group section sgroup_exists = exists_in_openstack(sgroup_show(sgroup)) if not sgroup_exists: create_sgroup_command = [ - 'server', 'group', 'create', '--policy', sgroup_policy, sgroup + "server", + "group", + "create", + "--policy", + sgroup_policy, + sgroup, ] - run_in_openstack(create_sgroup_command, rc_file) + run_in_openstack(create_sgroup_command, rc_file, with_tsocks) sgroup_output, _ = sgroup_show(sgroup) sgroup_object = json.loads(sgroup_output) - sgroup_id = sgroup_object['id'] + sgroup_id = sgroup_object["id"] # Port section - show_port_command = ['port', 'show', '-f', 'json', port_name] + show_port_command = ["port", "show", "-f", "json", port_name] port_exists = exists_in_openstack( - run_in_openstack(show_port_command, rc_file)) + run_in_openstack(show_port_command, rc_file, with_tsocks) + ) if not port_exists: - create_port_command = [ - 'port', 'create', '--network', network, port_name - ] - run_in_openstack(create_port_command, rc_file) + create_port_command = ["port", "create", "--network", network, port_name] + run_in_openstack(create_port_command, rc_file, with_tsocks) # Finaly set up is done, and we can create server create_server_command = [ - 'server', 'create', '--port', port_name, '--flavor', flavor, '--image', - image, '--boot-from-volume', volume_size, '--key-name', key_name, - '--hint', f'group={sgroup_id}', fqdn + "server", + "create", + "--port", + port_name, + "--flavor", + flavor, + "--image", + image, + "--boot-from-volume", + volume_size, + "--key-name", + key_name, + "--hint", + f"group={sgroup_id}", + fqdn, ] - run_in_openstack(create_server_command, rc_file) + run_in_openstack(create_server_command, rc_file, with_tsocks) timeout = 360 - succeed = wait_for_server(fqdn, rc_file, network, timeout=timeout) + succeed = wait_for_server(fqdn, rc_file, network, with_tsocks, timeout=timeout) if not succeed: print(f"Wait for server: {fqdn} timed out in {timeout}") return 4 if dns_record: - output, _ = show_server(fqdn, rc_file) - addresses = json.loads(output)['addresses'][network] + output, _ = show_server(fqdn, rc_file, with_tsocks) + addresses = json.loads(output)["addresses"][network] print(get_dns_records(addresses, fqdn)) else: - print(show_server(fqdn, rc_file, format)) + print(show_server(fqdn, rc_file, with_tsocks, format)) return 0 def delete(options: dict) -> int: - fqdn = options['fqdn'] - rc_file = options['rc_file'] - preserve_port = options['preserve_port'] - preserve_cosmos_overlay = options['preserve_cosmos_overlay'] - preserve_disk = options['preserve_disk'] + fqdn = options["fqdn"] + rc_file = options["rc_file"] + preserve_port = options["preserve_port"] + preserve_cosmos_overlay = options["preserve_cosmos_overlay"] + preserve_disk = options["preserve_disk"] + with_tsocks = options["with_tsocks"] # Check if server exists - server_output_error = show_server(fqdn, rc_file) + server_output_error = show_server(fqdn, rc_file, with_tsocks) server_exists = exists_in_openstack(server_output_error) # We will bail out early if the server does not allready exists if not server_exists: print(f"No server called {fqdn} exists.") return 0 - delete_server_command = ['server', 'delete', fqdn] - run_in_openstack(delete_server_command, rc_file) + delete_server_command = ["server", "delete", fqdn] + run_in_openstack(delete_server_command, rc_file, with_tsocks) print(f"Deleted server: {fqdn}") if not preserve_disk: server_object = json.loads(server_output_error[0]) - for disk in server_object['volumes_attached']: - delete_disk_command = ['volume', 'delete', disk['id']] - run_in_openstack(delete_disk_command, rc_file) + for disk in server_object["volumes_attached"]: + delete_disk_command = ["volume", "delete", disk["id"]] + run_in_openstack(delete_disk_command, rc_file, with_tsocks) # Port section if not preserve_port: # FIXME: we don't support setting the portname, we just use the fqdn and append -port to it - port_name = options['fqdn'].replace('.', '-') + '-port' - show_port_command = ['port', 'show', '-f', 'json', port_name] + port_name = options["fqdn"].replace(".", "-") + "-port" + show_port_command = ["port", "show", "-f", "json", port_name] port_exists = exists_in_openstack( - run_in_openstack(show_port_command, rc_file)) + run_in_openstack(show_port_command, rc_file, with_tsocks) + ) if port_exists: - delete_port_command = ['port', 'delete', port_name] - run_in_openstack(delete_port_command, rc_file) + delete_port_command = ["port", "delete", port_name] + run_in_openstack(delete_port_command, rc_file, with_tsocks) print(f"Deleted port: {port_name}") dir = os.path.dirname(os.path.abspath(__file__)) overlay = path_join(dir, fqdn) @@ -509,29 +563,34 @@ def delete(options: dict) -> int: def show(options: dict) -> None: - if options['format'] == 'bind': + if options["format"] == "bind": # We assume that only public addresses go in dns addresses = json.loads( - show_server(options['fqdn'], - options['rc_file'])[0])['addresses']['public'] - print(get_dns_records(addresses, options['fqdn'])) + show_server(options["fqdn"], options["rc_file"], options["with_tsocks"])[0] + )["addresses"]["public"] + print(get_dns_records(addresses, options["fqdn"])) else: print( - show_server(options['fqdn'], options['rc_file'], - options['format'])) + show_server( + options["fqdn"], + options["rc_file"], + options["with_tsocks"], + options["format"], + ) + ) def main() -> int: # Make sure we have openstack app credentials matching the naming scheme -- options = get_options() - options['rc_file'] = setup_rc_file(options) - command = options['command'] - if command == 'create': + options["rc_file"] = setup_rc_file(options) + command = options["command"] + if command == "create": return create(options) - if command == 'delete': + if command == "delete": return delete(options) - if command == 'show': + if command == "show": show(options) return 0