From e4ec129e4f559bc44a0be62f6a2f3a03a85f7379 Mon Sep 17 00:00:00 2001 From: Teresa Noviello Date: Mon, 1 Nov 2021 12:11:58 +0000 Subject: [PATCH] Added to the load test script documentation and the new option of verbose mode, which triggers also the load debug, at the time only tracking the php service's rating requests. Rating requests are logged into a log file and formatted into a CSV file --- .gitignore | 4 ++- load-gen/Dockerfile | 1 + load-gen/entrypoint.sh | 2 ++ load-gen/load-gen.sh | 18 +++++++++++-- load-gen/logs/placeholder.txt | 0 load-gen/robot-shop.py | 44 ++++++++++++++++++++++++++++++-- load-gen/utilities/CSVReader.py | 19 ++++++++++++++ load-gen/utilities/CSVWriter.py | 45 +++++++++++++++++++++++++++++++++ 8 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 load-gen/logs/placeholder.txt create mode 100644 load-gen/utilities/CSVReader.py create mode 100644 load-gen/utilities/CSVWriter.py diff --git a/.gitignore b/.gitignore index d8ed3c8..1cdbff0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .DS_Store *-private.* vendor/ -Gopkg.lock \ No newline at end of file +Gopkg.lock +load-gen/utilities/__pycache__ +load-gen/logs/php* diff --git a/load-gen/Dockerfile b/load-gen/Dockerfile index 54cbb5d..46b18f1 100644 --- a/load-gen/Dockerfile +++ b/load-gen/Dockerfile @@ -16,6 +16,7 @@ RUN pip install -r requirements.txt COPY entrypoint.sh /load/ COPY robot-shop.py /load/ +COPY utilities /load/ CMD ["./entrypoint.sh"] diff --git a/load-gen/entrypoint.sh b/load-gen/entrypoint.sh index aa56a03..6a546fc 100755 --- a/load-gen/entrypoint.sh +++ b/load-gen/entrypoint.sh @@ -35,6 +35,8 @@ else unset TIME fi +mkdir -p logs; + echo "Starting $CLIENTS clients for ${RUN_TIME:-ever}" if [ "$SILENT" -eq 1 ] then diff --git a/load-gen/load-gen.sh b/load-gen/load-gen.sh index 49abda4..05f98a6 100755 --- a/load-gen/load-gen.sh +++ b/load-gen/load-gen.sh @@ -15,6 +15,9 @@ HOST="http://localhost:8080" # Error flag ERROR=0 +# Verbose mode +LOAD_DEBUG=1 + # Daemon flag DAEMON="-it" SILENT=0 @@ -24,7 +27,8 @@ USAGE="\ loadgen.sh e - error flag -d - run in background +v - verbose mode. It implies the setting of LOAD_DEBUG and of error flag +d - run in background. It implies the setting of SILENT mode, so the stdout will not be floaded. If -v is used, the load debug will still be printed in the stdout, even if the stdout is suppressed. n - number of clients t - time to run n clients h - target host @@ -42,12 +46,16 @@ eval $(egrep '[A-Z]+=' ../.env) echo "Repo $REPO" echo "Tag $TAG" -while getopts 'edn:t:h:' OPT +while getopts 'edvn:t:h:' OPT do case $OPT in e) ERROR=1 ;; + v) + LOAD_DEBUG=1 + ERROR=1 + ;; d) DAEMON="-d" SILENT=1 @@ -92,9 +100,14 @@ do esac done +rm -rf logs/* docker run \ $DAEMON \ --name loadgen \ + --volume ${PWD}/robot-shop.py:/load/robot-shop.py \ + --volume ${PWD}/entrypoint.sh:/load/entrypoint.sh \ + --volume ${PWD}/utilities:/load/utilities \ + --volume ${PWD}/logs:/load/logs \ --rm \ --network=host \ -e "HOST=$HOST" \ @@ -102,5 +115,6 @@ docker run \ -e "RUN_TIME=$RUN_TIME" \ -e "SILENT=$SILENT" \ -e "ERROR=$ERROR" \ + -e "LOAD_DEBUG=$LOAD_DEBUG" \ ${REPO}/rs-load:${TAG} diff --git a/load-gen/logs/placeholder.txt b/load-gen/logs/placeholder.txt new file mode 100644 index 0000000..e69de29 diff --git a/load-gen/robot-shop.py b/load-gen/robot-shop.py index 21adae9..a3bbecc 100644 --- a/load-gen/robot-shop.py +++ b/load-gen/robot-shop.py @@ -2,8 +2,11 @@ import os import random from locust import HttpUser, task, between +from utilities.CSVWriter import CSVWriter from random import choice from random import randint +from sys import argv +from datetime import date class UserBehavior(HttpUser): wait_time = between(2, 10) @@ -26,9 +29,26 @@ class UserBehavior(HttpUser): "60.242.161.215" ] + phplog = None + php_services_api_prefix = '/api/ratings/api' + php_service_rate = '/rate' + php_service_fetch = '/fetch' + php_fieldnames = ['REQTYPE', 'SERVICE', 'INPUT', 'HEADER'] + my_csv_writer = None + def on_start(self): """ on_start is called when a Locust start before any task is scheduled """ print('Starting') + print('LOAD_DEBUG: ', os.environ.get("LOAD_DEBUG")) + + if os.environ.get('LOAD_DEBUG') == '1': + print("ARGS ARE:\n\"") + print("\n".join(argv)) + print('End of ARGS printing\n') + print('on start. php_fieldnames:{}', format(self.php_fieldnames)) + + self.my_csv_writer = CSVWriter("logs/php_services_calls.csv", self.php_fieldnames) + self.phplog = open("logs/phplog.txt", "a") @task def login(self): @@ -44,6 +64,7 @@ class UserBehavior(HttpUser): @task def load(self): + self.phplog.write('new user, new load task\n') fake_ip = random.choice(self.fake_ip_addresses) self.client.get('/', headers={'x-forwarded-for': fake_ip}) @@ -61,12 +82,31 @@ class UserBehavior(HttpUser): if item['instock'] != 0: break + self.phplog.write('fake_ip: {}\n'.format(fake_ip)) + + headers={'x-forwarded-for': fake_ip} # vote for item if randint(1, 10) <= 3: - self.client.put('/api/ratings/api/rate/{}/{}'.format(item['sku'], randint(1, 5)), headers={'x-forwarded-for': fake_ip}) + ratevalue = randint(1, 5) + put_rate_api_str = '{}{}/{}/{}'.format(self.php_services_api_prefix, self.php_service_rate, item['sku'], ratevalue ) + if os.environ.get('LOAD_DEBUG') == '1': + self.phplog.write('ratevalue: {}\n'.format(ratevalue)) + self.phplog.write('item: {}\n'.format(item['sku'])) + self.phplog.write('put_rate_api_str: {}\n'.format(put_rate_api_str)) + self.phplog.flush() + self.my_csv_writer.writerow({'REQTYPE': 'PUT', 'SERVICE': '{}'.format(self.php_service_rate), 'INPUT': '{}/{}'.format(item['sku'], ratevalue ), 'HEADER': '{}'.format(headers)}) + self.client.put(put_rate_api_str, headers) self.client.get('/api/catalogue/product/{}'.format(item['sku']), headers={'x-forwarded-for': fake_ip}) - self.client.get('/api/ratings/api/fetch/{}'.format(item['sku']), headers={'x-forwarded-for': fake_ip}) + + get_rate_api_str = '{}{}/{}'.format(self.php_services_api_prefix, self.php_service_fetch, item['sku']) + if os.environ.get('LOAD_DEBUG') == '1': + self.phplog.write('item: {}\n'.format(item['sku'])) + self.phplog.write('get_rate_api_str: {}\n'.format(get_rate_api_str)) + self.phplog.flush() + self.my_csv_writer.writerow({'REQTYPE': 'GET', 'SERVICE': '{}'.format(self.php_service_fetch), 'INPUT': '{}'.format(item['sku']), 'HEADER': '{}'.format(headers) }) + self.client.get(get_rate_api_str, headers={'x-forwarded-for': fake_ip}) + self.client.get('/api/cart/add/{}/{}/1'.format(uniqueid, item['sku']), headers={'x-forwarded-for': fake_ip}) cart = self.client.get('/api/cart/cart/{}'.format(uniqueid), headers={'x-forwarded-for': fake_ip}).json() diff --git a/load-gen/utilities/CSVReader.py b/load-gen/utilities/CSVReader.py new file mode 100644 index 0000000..ab40c1f --- /dev/null +++ b/load-gen/utilities/CSVReader.py @@ -0,0 +1,19 @@ +import csv + +# Refs: +# [Tutorial1] https://rharshad.com/locust-load-test/ +class CSVReader: + def __init__(self, filepath): + try: + file = open(filepath) + except TypeError: + pass + self.file = file + self.reader = csv.reader(file) + + def __next__(self): + try: + return next(self.reader) + except StopIteration: + self.file.seek(0, 0) + return next(self.reader) diff --git a/load-gen/utilities/CSVWriter.py b/load-gen/utilities/CSVWriter.py new file mode 100644 index 0000000..72fc942 --- /dev/null +++ b/load-gen/utilities/CSVWriter.py @@ -0,0 +1,45 @@ +import os +import csv +from datetime import date + +class CSVWriter: + def __init__(self, filepath, custom_fields=[]): + if os.environ.get('LOAD_DEBUG') == '1': + print('CSVWriter init/args: filepath:\'{}\' custom_fields\'{}\''.format(filepath,custom_fields)) + + try: + file = open(filepath,'a') + except TypeError as typeErr: + print(f"CSVWriter init/Unexpected {err=}, {type(typeErr)=}") + raise + + print('CSVWriter init/pass') + self.file = file + + if not custom_fields: + print('CSVWriter init/empty params') + else: + self.fieldnames = custom_fields + + try: + self.writer = csv.DictWriter(self.file, self.fieldnames) + self.writer.writeheader() + except BaseException as err: + print(f"CSVWriter init/Unexpected {err=}, {type(err)=}") + raise + + self.file.flush() + + def writerow(self, row: dict): + if os.environ.get('LOAD_DEBUG') == '1': + print('CSVWriter/writerow/args: {}',format(row)) + + if not row: + print('CSVWriter/writerow/empty arg') + else: + try: + self.writer.writerow(row) + self.file.flush() + except BaseException as err: + print(f"CSVWriter/writerow/Unexpected {err=}, {type(err)=}") + raise