Renamed general php call log to calls.log and redirected there only the php-related info. Added rotation to calls.log. Updated the documentation.

.gitignore

@@ -2,4 +2,9 @@
 .DS_Store
 *-private.*
 vendor/
-Gopkg.lock
+Gopkg.lock
+load-gen/utilities/__pycache__
+load-gen/logs/*.log
+load-gen/logs/*.log.[0-9]*
+load-gen/logs/*.csv
+agent/logs

load-gen/Dockerfile

@@ -16,6 +16,7 @@ RUN pip install -r requirements.txt
 
 COPY entrypoint.sh /load/
 COPY robot-shop.py /load/
+COPY utilities /load/
 
 CMD ["./entrypoint.sh"]
 

load-gen/README.md

@@ -14,7 +14,20 @@ $ ./load-gen.sh
 
 Runs the load generation script against the application started with `docker-compose up` . There are various command line options to configure the load.
 
-Alternatively, you can run the Container from Docker Hub directly on one of the nodes having access to the web service:
+The script must be run in the load-gen directory. It logs all the php API calls into the file logs/php_services_calls.csv.
+
+This command launches the load generator to run undefinitely (i.e. without time limits), simulating 5 clients calling the API reachable at the default URL http://localhost:8080:
+
+```shell
+$ ./load-gen.sh \
+-h http://host:port/
+-n 5 \
+-v
+```
+
+The command also logs comprehensive details of all the PHP API called in the file logs/calls.log, triggered by the option `-v` .
+
+Alternatively, you can run the Container from Docker Hub directly on one of the nodes having access to the web service. Here there is an example of how to do it and an explanation for the variables involved:
 
 ```shell
 $ docker run \
@@ -22,11 +35,14 @@ $ docker run \
 --rm \
 --name="loadgen" \
 --network=host \
+--volume ${PWD}/logs:/load/logs \
+--volume ${PWD}/utilities:/load/utilities \
 -e "HOST=http://host:8080/"
 -e "NUM_CLIENTS=5" \
 -e "RUN_TIME=1h30m" \
 -e "ERROR=1" \
 -e "SILENT=1" \
+-e "LOAD_DEBUG=0" \
 robotshop/rs-load
 ```
 
@@ -36,7 +52,11 @@ Set the following environment variables to configure the load:
 * NUM_CLIENTS - How many simultaneous load scripts to run, the bigger the number the bigger the load. The default is 1
 * RUN_TIME - For NUM_CLIENTS greater than 1 the duration to run. If not set, load is run for ever with NUM_CLIENTS. See below.
 * ERROR - Set this to 1 to have erroroneous calls made to the payment service.
-* SILENT - Set this to 1 to surpress the very verbose output from the script. This is a good idea if you're going to run load for more than a few minutes.
+* SILENT - Set this to 1 to surpress the very verbose output to the stdout from the script. This is a good idea if you're going to run load for more than a few minutes.
+* LOAD_DEBUG - Set this to 1 to enable the output of every API call produced from the script into the log file logs/calls.log. This is a good idea if you're going to investigate over occurred events during load generation.
+
+The load generator logs all the PHP API calls into the file logs/php_services_calls.csv, despite the variables SILENT and LOAD_DEBUG being set, respectively, to 1 and 0.
+The content of the directory logs is cleaned everytime the script load-gen.sh is called.
 
 ## Kubernetes
 
@@ -67,4 +87,4 @@ $ ./load-gen.sh \
 -t 1h30m
 ```
 
-The load will be run with `10` clients for `1h30m` before dropping down to `1` client for `1h30m` then looping back to `10` clients etc.
+The load will be run with `10` clients for `1h30m` before dropping down to `1` client for `1h30m` then looping back to `10` clients etc.

load-gen/entrypoint.sh

@@ -35,6 +35,8 @@ else
     unset TIME
 fi
 
+mkdir -p logs;
+
 echo "Starting $CLIENTS clients for ${RUN_TIME:-ever}"
 if [ "$SILENT" -eq 1 ]
 then

load-gen/load-gen.sh

@@ -15,6 +15,9 @@ HOST="http://localhost:8080"
 # Error flag
 ERROR=0
 
+# Verbose mode
+LOAD_DEBUG=1
+
 # Daemon flag
 DAEMON="-it"
 SILENT=0
@@ -24,7 +27,8 @@ USAGE="\
 loadgen.sh
 
 e - error flag
-d - run in background
+v - verbose mode. It implies the setting of LOAD_DEBUG and of error flag. The load debug will still be printed in the stdout, even if the stdout is suppressed.
+d - run in background. It implies the setting of SILENT mode, so the stdout will not be floaded. If -v is used, the load debug will still be printed in the stdout, even if the stdout is suppressed.
 n - number of clients
 t - time to run n clients
 h - target host
@@ -42,12 +46,16 @@ eval $(egrep '[A-Z]+=' ../.env)
 echo "Repo $REPO"
 echo "Tag $TAG"
 
-while getopts 'edn:t:h:' OPT
+while getopts 'edvn:t:h:' OPT
 do
     case $OPT in
         e)
             ERROR=1
             ;;
+        v)
+            LOAD_DEBUG=1
+            ERROR=1
+            ;;
         d)
             DAEMON="-d"
             SILENT=1
@@ -92,9 +100,13 @@ do
     esac
 done
 
+rm -rf logs/*
+
 docker run \
     $DAEMON \
     --name loadgen \
+    --volume ${PWD}/logs:/load/logs \
+    --volume ${PWD}/utilities:/load/utilities \
     --rm \
     --network=host \
     -e "HOST=$HOST" \
@@ -102,5 +114,6 @@ docker run \
     -e "RUN_TIME=$RUN_TIME" \
     -e "SILENT=$SILENT" \
     -e "ERROR=$ERROR" \
+    -e "LOAD_DEBUG=$LOAD_DEBUG" \
     ${REPO}/rs-load:${TAG}
 

load-gen/robot-shop.py

@@ -1,9 +1,13 @@
 import os
 import random
+import logging
 
 from locust import HttpUser, task, between
+from utilities.CSVWriter import CSVWriter
 from random import choice
 from random import randint
+from sys import argv
+from datetime import date
 
 class UserBehavior(HttpUser):
     wait_time = between(2, 10)
@@ -26,9 +30,41 @@ class UserBehavior(HttpUser):
         "60.242.161.215"
     ]
 
+    logger = None
+    rthandler = None
+    formatter = None
+    php_services_api_prefix = '/api/ratings/api'
+    php_service_rate = '/rate'
+    php_service_fetch = '/fetch'
+    php_fieldnames = ['REQTYPE', 'SERVICE', 'INPUT', 'HEADER', 'ERRFLAG']
+    my_csv_writer = None
+
     def on_start(self):
         """ on_start is called when a Locust start before any task is scheduled """
         print('Starting')
+        print("ARGS ARE:\n\"")
+        print("\n".join(argv))
+        print('End of ARGS \n')
+
+        for handler in logging.root.handlers[:]:
+            logging.root.removeHandler(handler)
+
+        self.logger = logging.getLogger('simple_rotating_logger')
+        self.rthandler = logging.handlers.RotatingFileHandler(filename='logs/calls.log', mode='a', maxBytes=5242880, backupCount=20, encoding='utf-8')
+        self.formatter = logging.Formatter('%(asctime)s [%(levelname)s]:%(message)s')
+        self.rthandler.setFormatter(self.formatter)
+        self.logger.addHandler(self.rthandler)
+
+        if os.environ.get('LOAD_DEBUG') == '1':
+            self.logger.setLevel(logging.DEBUG)
+        else:
+            self.logger.setLevel(logging.WARNING)
+
+        self.logger.info('Starting')
+        self.logger.info('LOAD_DEBUG: %s', os.environ.get("LOAD_DEBUG"))
+        self.logger.info('on start. php_fieldnames: %s', format(self.php_fieldnames))
+
+        self.my_csv_writer = CSVWriter("logs/php_services_calls.csv", self.php_fieldnames)
 
     @task
     def login(self):
@@ -44,6 +80,7 @@ class UserBehavior(HttpUser):
 
     @task
     def load(self):
+        self.logger.info('new user, new load task\n')
         fake_ip = random.choice(self.fake_ip_addresses)
 
         self.client.get('/', headers={'x-forwarded-for': fake_ip})
@@ -61,12 +98,34 @@ class UserBehavior(HttpUser):
                 if item['instock'] != 0:
                     break
 
+            headers={'x-forwarded-for': fake_ip}
             # vote for item
             if randint(1, 10) <= 3:
-                self.client.put('/api/ratings/api/rate/{}/{}'.format(item['sku'], randint(1, 5)), headers={'x-forwarded-for': fake_ip})
+                ratevalue = randint(1, 5)
+                put_rate_api_str = '{}{}/{}/{}'.format(self.php_services_api_prefix, self.php_service_rate, item['sku'], ratevalue )
+                self.logger.info('item: {} ratevalue: {} put_rate_api_str: {} by: {}\n'.format(item['sku'], ratevalue, put_rate_api_str, fake_ip))
+                try:
+                    self.client.put(put_rate_api_str, headers)
+                    self.my_csv_writer.writerow({'REQTYPE': 'PUT', 'SERVICE': '{}'.format(self.php_service_rate), 'INPUT': '{}/{}'.format(item['sku'], ratevalue ), 'HEADER': '{}'.format(headers), 'ERRFLAG': '{}'.format("")})
+                except BaseException as err:
+                    self.logger.warnign("Last call generated an error")
+                    self.logger.exception()
+                    self.my_csv_writer.writerow({'REQTYPE': 'PUT', 'SERVICE': '{}'.format(self.php_service_rate), 'INPUT': '{}/{}'.format(item['sku'], ratevalue ), 'HEADER': '{}'.format(headers), 'ERRFLAG': '{}'.format(err)})
+                    pass
 
             self.client.get('/api/catalogue/product/{}'.format(item['sku']), headers={'x-forwarded-for': fake_ip})
-            self.client.get('/api/ratings/api/fetch/{}'.format(item['sku']), headers={'x-forwarded-for': fake_ip})
+
+            get_rate_api_str = '{}{}/{}'.format(self.php_services_api_prefix, self.php_service_fetch, item['sku'])
+            self.logger.info('item: {} get_rate_api_str: {} by: {}\n'.format(item['sku'], get_rate_api_str, fake_ip))
+            try:
+                self.client.get(get_rate_api_str, headers={'x-forwarded-for': fake_ip})
+                self.my_csv_writer.writerow({'REQTYPE': 'GET', 'SERVICE': '{}'.format(self.php_service_fetch), 'INPUT': '{}'.format(item['sku']), 'HEADER': '{}'.format(headers), 'ERRFLAG': '{}'.format("") })
+            except BaseException as err:
+                self.logger.warnign("Last call generated an error")
+                self.logger.exception()
+                self.my_csv_writer.writerow({'REQTYPE': 'GET', 'SERVICE': '{}'.format(self.php_service_fetch), 'INPUT': '{}'.format(item['sku']), 'HEADER': '{}'.format(headers), 'ERRFLAG': '{}'.format(err) })
+                pass
+
             self.client.get('/api/cart/add/{}/{}/1'.format(uniqueid, item['sku']), headers={'x-forwarded-for': fake_ip})
 
         cart = self.client.get('/api/cart/cart/{}'.format(uniqueid), headers={'x-forwarded-for': fake_ip}).json()

load-gen/utilities/CSVReader.py

@@ -0,0 +1,19 @@
+import csv
+
+# Refs:
+# [Tutorial1] https://rharshad.com/locust-load-test/
+class CSVReader:
+    def __init__(self, filepath):
+        try:
+            file = open(filepath)
+        except TypeError:
+            pass
+        self.file = file
+        self.reader = csv.reader(file)
+
+    def __next__(self):
+        try:
+            return next(self.reader)
+        except StopIteration:
+            self.file.seek(0, 0)
+            return next(self.reader)

load-gen/utilities/CSVWriter.py

@@ -0,0 +1,45 @@
+import os
+import csv
+from datetime import date
+
+class CSVWriter:
+    def __init__(self, filepath, custom_fields=[]):
+        if os.environ.get('LOAD_DEBUG') == '1':
+            print('CSVWriter init/args: filepath:\'{}\' custom_fields\'{}\''.format(filepath,custom_fields))
+
+        try:
+            file = open(filepath,'a')
+        except TypeError as typeErr:
+            print(f"CSVWriter init/Unexpected {err=}, {type(typeErr)=}")
+            raise
+
+        print('CSVWriter init/pass')
+        self.file = file
+
+        if not custom_fields:
+            print('CSVWriter init/empty params')
+        else:
+            self.fieldnames = custom_fields
+
+        try:
+            self.writer = csv.DictWriter(self.file, self.fieldnames)
+            self.writer.writeheader()
+        except BaseException as err:
+            print(f"CSVWriter init/Unexpected {err=}, {type(err)=}")
+            raise
+
+        self.file.flush()
+
+    def writerow(self, row: dict):
+        if os.environ.get('LOAD_DEBUG') == '1':
+            print('CSVWriter/writerow/args: {}',format(row))
+
+        if not row:
+            print('CSVWriter/writerow/empty arg')
+        else:
+            try:
+                self.writer.writerow(row)
+                self.file.flush()
+            except BaseException as err:
+                print(f"CSVWriter/writerow/Unexpected {err=}, {type(err)=}")
+                raise