From ad0c25a22b897f92161ce673b4d76f66ae2df049 Mon Sep 17 00:00:00 2001 From: Martin Blazik Date: Fri, 4 Dec 2020 20:14:59 +0100 Subject: [PATCH] dcm add more commands Commands: on, wait, check an start all tiger services in right order --- bin/dcm | 215 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 132 insertions(+), 83 deletions(-) diff --git a/bin/dcm b/bin/dcm index 3795203..d7352d7 100755 --- a/bin/dcm +++ b/bin/dcm @@ -9,43 +9,26 @@ import yaml import pprint import os import re -from sys import argv +from sys import argv, exit from time import time, sleep from pprint import PrettyPrinter - - -class Dependecies: - def __init__(self, deps): - self.deps = deps - - @staticmethod - def load(yaml_filename): - deps = {} - with open(yaml_filename) as file: - dc = yaml.load(file, Loader=yaml.FullLoader) - for (name, conf) in dc['services'].items(): - depends_on = conf.get('depends_on', []) - deps[name] = depends_on - return deps - - def find(self, service): - all = set([service]) - if service in self.deps: - services = self.deps[service] - new = set(services) - all - if new: - print(f"{service}: {new}") - all.update(services) - del self.deps[service] - for one in new: - all.update(self.find(one)) - return all +from operator import itemgetter def echo(text): print(">> " + text) +def print_list(lst): + for item in lst: + print(item) + + +def error(text): + print(text) + exit(1) + + def system(cmd): return subprocess.call(cmd, shell=True) @@ -58,6 +41,10 @@ def prefix_join(prefix, array): return " ".join([prefix + " " + item for item in array]) +def join_args(args): + return " ".join(args) + + def dc_files(files): return prefix_join("-f", files) @@ -70,8 +57,8 @@ def load_dc_file(dc_file): def dc_cmd(args): # split join - args = " ".join(args) - return " ".join([DOCKER_COMPOSE, DC_FILES_ARGS, args]) + args = join_args(args) + return join_args([DOCKER_COMPOSE, DC_FILES_ARGS, args]) def dc_run(args): @@ -91,52 +78,60 @@ def dc_services(dc_file): def dump_services(files): - for filename in files: - for service in dc_services(filename): - print(service) + services = [(filename, service) for filename in files for service in dc_services(filename)] + services = map(itemgetter(1), services) + #services = sorted(services) + print_list(services) -def dc_deps(dc_files, service): +def deps_load(yaml_filename): + deps = {} + with open(yaml_filename) as file: + dc = yaml.load(file, Loader=yaml.FullLoader) + for (name, conf) in dc['services'].items(): + depends_on = conf.get('depends_on', []) + deps[name] = depends_on + return deps + + +def deps_find(service, deps): + all = set([service]) + if service in deps: + services = deps[service] + new = set(services) - all + if new: + new_list = " ".join(new) + print(f"{service} >> {new_list}") + all.update(services) + del deps[service] + for one in new: + all.update(deps_find(one, deps)) + return all + + +def dc_deps(dc_files, args): + service = args[0] pp = PrettyPrinter(indent=2) for dc_file in dc_files: - dep = Dependecies.load(dc_file) - if service in dep: - all = sorted(Dependecies(dep).find(service)) - print(all) + deps = deps_load(dc_file) + if service in deps: + all = sorted(deps_find(service, deps)) + print("ALL >> " + " ".join(all)) return print(f"Service {service} not found") - def dump_files(files): for file in files: print(file) -# AFM exec API: curl -fs -o /dev/null http://localhost:9001/actuator/health/readiness -# AQE: /opt/bin/aqe ping -# md-api-new: curl -fs -o /dev/null http://localhost:9008/actuator/health/readiness -# potgres: s6-setuidgid postgres; /usr/bin/pg_isready -# redis: /usr/bin/redis-cli ping -# result-cache: curl -fs -o /dev/null http://localhost:9041/actuator/health/readiness -# scan-model: curl -fs -o /dev/null http://localhost:9061/actuator/health/readiness -# sql-executor: curl -fs -o /dev/null http://localhost:9101/actuator/health/readiness - - def curl_check(uri): cmd = "curl -f -s -o /dev/null " + uri echo(cmd) return system(cmd) == 0 -def check_pulsar(): - return curl_check("http://localhost:8080/admin/v2/tenants/public") - - -def check_postgres(): - return system("/usr/bin/pg_isready") == 0 - - def wait_for(check, delay, duration): start = time() while time() - start < duration: @@ -148,22 +143,58 @@ def wait_for(check, delay, duration): return False -def up_and_wait(service, check, delay, duration): - dc_run(["up", "-d", service]) - started = wait_for(check, delay, duration) +def service_wait(service, check, delay = 10, duration = 120): echo(f"WAIT FOR {service}") + started = wait_for(check, delay, duration) if started: echo(f"READY {service}") else: echo(f"BAD {service}") -def start_tiger(): - up_and_wait("pulsar", check_pulsar, 5, 60) - up(["redis", "jaeger", "router"]) - up_and_wait("postgres", check_postgres, 5, 60) - up(["aqe", "afm-exec-api", "result-cache", "scan-model", "sql-executor", "metadata-api-new"]) - +def get_check(service): + checker = SERVICES.get(service, None) + if checker is None: + error(f"No checker for {service}") + else: + return checker + + +def dc_check(args): + for service in args: + check = get_check(service) + running = check() + if running: + print(f"{service} ON") + else: + print(f"{service} OFF") + + +def dc_up(services): + dc_run(["up", "-d", join_args(services)]) + + +def dc_wait(args): + for service in args: + check = get_check(service) + service_wait(service, check) + + +def dc_on(args): + for service in args: + if service == "tiger": + tiger_on() + else: + check = get_check(service) + dc_up([service]) + service_wait(service, check) + + +def tiger_on(): + dc_on(["pulsar", "postgres"]) + dc_up(["redis", "jaeger", "router"]) + dc_on(["result-cache", "metadata-api", "aqe", "sql-executor", "scan-model", "afm-exec-api"]) + def try_load_list(name, default): if name in os.environ: @@ -179,28 +210,46 @@ DC_FILES = try_load_list("DCF", DEFAULT_DCF) DC_FILES_ARGS = dc_files(DC_FILES) SERVICES = { - "pulsar": lambda: up_and_wait("pulsar", check_pulsar, 10, 120), - "postgres": lambda: up_and_wait("postgres", check_postgres, 10, 120), - "data-loader": None, # co chci testovat? ze ma Exit code 0 + "pulsar": + lambda: + curl_check("http://localhost:8080/admin/v2/tenants/public"), + "postgres": + lambda: + system("/usr/bin/pg_isready") == 0, + "redis": + lambda: + system("/usr/bin/redis-cli ping") == 0, + # AQE: /opt/bin/aqe ping + "aqe": + lambda: + True, + "metadata-api": + lambda: + curl_check("http://localhost:9008/actuator/health/readiness"), + "afm-exec-api": + lambda: + curl_check("http://localhost:9001/actuator/health/readiness"), + "result-cache": + lambda: + curl_check("http://localhost:9041/actuator/health/readiness"), + "scan-model": + lambda: + curl_check("http://localhost:9061/actuator/health/readiness"), + "sql-executor": + lambda: + curl_check("http://localhost:9101/actuator/health/readiness"), + "data-loader": + None, # co chci testovat? ze ma Exit code 0 } -DEFAULT_TIGER_SERVICES = [ - "afm-exec-api", - "metadata-api-new", - "aqe", - "result-cache", - "scan-model", - "sql-executor" -] -TIGER_SERVICES = try_load_list("SERVICES", DEFAULT_TIGER_SERVICES) - COMMANDS = { "files": lambda args: dump_files(DC_FILES), "services": lambda args: dump_services(DC_FILES), "log": dc_log, - "deps": lambda args: dc_deps(DC_FILES, args[0]), - "wait": None, - "up-wait": lambda args: SERVICES[args[0]](), + "deps": lambda args: dc_deps(DC_FILES, args), + "check": dc_check, + "wait": dc_wait, + "on": dc_on, }