lachtan/linux-workspace
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dcm add more commands

Browse Source 
Commands: on, wait, check
an start all tiger services in right order
This commit is contained in:
Martin Blazik
2020-12-04 20:14:59 +01:00
parent a4f936a6db
commit ad0c25a22b
1 changed files with 132 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
bin/dcm
View File

@@ -9,43 +9,26 @@ import yaml
import pprint
import os
import re
from sys import argv
from sys import argv, exit
from time import time, sleep
from pprint import PrettyPrinter
class Dependecies:
def __init__(self, deps):
self.deps = deps
@staticmethod
def load(yaml_filename):
deps = {}
with open(yaml_filename) as file:
dc = yaml.load(file, Loader=yaml.FullLoader)
for (name, conf) in dc['services'].items():
depends_on = conf.get('depends_on', [])
deps[name] = depends_on
return deps
def find(self, service):
all = set([service])
if service in self.deps:
services = self.deps[service]
new = set(services) - all
if new:
print(f"{service}: {new}")
all.update(services)
del self.deps[service]
for one in new:
all.update(self.find(one))
return all
from operator import itemgetter
def echo(text):
print(">> " + text)
def print_list(lst):
for item in lst:
print(item)
def error(text):
print(text)
exit(1)
def system(cmd):
return subprocess.call(cmd, shell=True)
@@ -58,6 +41,10 @@ def prefix_join(prefix, array):
return " ".join([prefix + " " + item for item in array])
def join_args(args):
return " ".join(args)
def dc_files(files):
return prefix_join("-f", files)
@@ -70,8 +57,8 @@ def load_dc_file(dc_file):
def dc_cmd(args):
# split join
args = " ".join(args)
return " ".join([DOCKER_COMPOSE, DC_FILES_ARGS, args])
args = join_args(args)
return join_args([DOCKER_COMPOSE, DC_FILES_ARGS, args])
def dc_run(args):
@@ -91,52 +78,60 @@ def dc_services(dc_file):
def dump_services(files):
for filename in files:
for service in dc_services(filename):
print(service)
services = [(filename, service) for filename in files for service in dc_services(filename)]
services = map(itemgetter(1), services)
#services = sorted(services)
print_list(services)
def dc_deps(dc_files, service):
def deps_load(yaml_filename):
deps = {}
with open(yaml_filename) as file:
dc = yaml.load(file, Loader=yaml.FullLoader)
for (name, conf) in dc['services'].items():
depends_on = conf.get('depends_on', [])
deps[name] = depends_on
return deps
def deps_find(service, deps):
all = set([service])
if service in deps:
services = deps[service]
new = set(services) - all
if new:
new_list = " ".join(new)
print(f"{service} >> {new_list}")
all.update(services)
del deps[service]
for one in new:
all.update(deps_find(one, deps))
return all
def dc_deps(dc_files, args):
service = args[0]
pp = PrettyPrinter(indent=2)
for dc_file in dc_files:
dep = Dependecies.load(dc_file)
if service in dep:
all = sorted(Dependecies(dep).find(service))
print(all)
deps = deps_load(dc_file)
if service in deps:
all = sorted(deps_find(service, deps))
print("ALL >> " + " ".join(all))
return
print(f"Service {service} not found")
def dump_files(files):
for file in files:
print(file)
# AFM exec API: curl -fs -o /dev/null http://localhost:9001/actuator/health/readiness
# AQE: /opt/bin/aqe ping
# md-api-new: curl -fs -o /dev/null http://localhost:9008/actuator/health/readiness
# potgres: s6-setuidgid postgres; /usr/bin/pg_isready
# redis: /usr/bin/redis-cli ping
# result-cache: curl -fs -o /dev/null http://localhost:9041/actuator/health/readiness
# scan-model: curl -fs -o /dev/null http://localhost:9061/actuator/health/readiness
# sql-executor: curl -fs -o /dev/null http://localhost:9101/actuator/health/readiness
def curl_check(uri):
cmd = "curl -f -s -o /dev/null " + uri
echo(cmd)
return system(cmd) == 0
def check_pulsar():
return curl_check("http://localhost:8080/admin/v2/tenants/public")
def check_postgres():
return system("/usr/bin/pg_isready") == 0
def wait_for(check, delay, duration):
start = time()
while time() - start < duration:
@@ -148,22 +143,58 @@ def wait_for(check, delay, duration):
return False
def up_and_wait(service, check, delay, duration):
dc_run(["up", "-d", service])
started = wait_for(check, delay, duration)
def service_wait(service, check, delay = 10, duration = 120):
echo(f"WAIT FOR {service}")
started = wait_for(check, delay, duration)
if started:
echo(f"READY {service}")
else:
echo(f"BAD {service}")
def start_tiger():
up_and_wait("pulsar", check_pulsar, 5, 60)
up(["redis", "jaeger", "router"])
up_and_wait("postgres", check_postgres, 5, 60)
up(["aqe", "afm-exec-api", "result-cache", "scan-model", "sql-executor", "metadata-api-new"])
def get_check(service):
checker = SERVICES.get(service, None)
if checker is None:
error(f"No checker for {service}")
else:
return checker
def dc_check(args):
for service in args:
check = get_check(service)
running = check()
if running:
print(f"{service} ON")
else:
print(f"{service} OFF")
def dc_up(services):
dc_run(["up", "-d", join_args(services)])
def dc_wait(args):
for service in args:
check = get_check(service)
service_wait(service, check)
def dc_on(args):
for service in args:
if service == "tiger":
tiger_on()
else:
check = get_check(service)
dc_up([service])
service_wait(service, check)
def tiger_on():
dc_on(["pulsar", "postgres"])
dc_up(["redis", "jaeger", "router"])
dc_on(["result-cache", "metadata-api", "aqe", "sql-executor", "scan-model", "afm-exec-api"])
def try_load_list(name, default):
if name in os.environ:
@@ -179,28 +210,46 @@ DC_FILES = try_load_list("DCF", DEFAULT_DCF)
DC_FILES_ARGS = dc_files(DC_FILES)
SERVICES = {
"pulsar": lambda: up_and_wait("pulsar", check_pulsar, 10, 120),
"postgres": lambda: up_and_wait("postgres", check_postgres, 10, 120),
"data-loader": None, # co chci testovat? ze ma Exit code 0
"pulsar":
lambda:
curl_check("http://localhost:8080/admin/v2/tenants/public"),
"postgres":
lambda:
system("/usr/bin/pg_isready") == 0,
"redis":
lambda:
system("/usr/bin/redis-cli ping") == 0,
# AQE: /opt/bin/aqe ping
"aqe":
lambda:
True,
"metadata-api":
lambda:
curl_check("http://localhost:9008/actuator/health/readiness"),
"afm-exec-api":
lambda:
curl_check("http://localhost:9001/actuator/health/readiness"),
"result-cache":
lambda:
curl_check("http://localhost:9041/actuator/health/readiness"),
"scan-model":
lambda:
curl_check("http://localhost:9061/actuator/health/readiness"),
"sql-executor":
lambda:
curl_check("http://localhost:9101/actuator/health/readiness"),
"data-loader":
None, # co chci testovat? ze ma Exit code 0
}
DEFAULT_TIGER_SERVICES = [
"afm-exec-api",
"metadata-api-new",
"aqe",
"result-cache",
"scan-model",
"sql-executor"
]
TIGER_SERVICES = try_load_list("SERVICES", DEFAULT_TIGER_SERVICES)
COMMANDS = {
"files": lambda args: dump_files(DC_FILES),
"services": lambda args: dump_services(DC_FILES),
"log": dc_log,
"deps": lambda args: dc_deps(DC_FILES, args[0]),
"wait": None,
"up-wait": lambda args: SERVICES[args[0]](),
"deps": lambda args: dc_deps(DC_FILES, args),
"check": dc_check,
"wait": dc_wait,
"on": dc_on,
}