Added to the load test script documentation and the new option of verbose mode, which triggers also the load debug, at the time only tracking the php service's rating requests. Rating requests are logged into a log file and formatted into a CSV file

.gitignore

@@ -2,4 +2,6 @@
 .DS_Store
 *-private.*
 vendor/
-Gopkg.lock
+Gopkg.lock
+load-gen/utilities/__pycache__
+load-gen/logs/php*

load-gen/Dockerfile

@@ -16,6 +16,7 @@ RUN pip install -r requirements.txt
 
 COPY entrypoint.sh /load/
 COPY robot-shop.py /load/
+COPY utilities /load/
 
 CMD ["./entrypoint.sh"]
 

load-gen/entrypoint.sh

@@ -35,6 +35,8 @@ else
     unset TIME
 fi
 
+mkdir -p logs;
+
 echo "Starting $CLIENTS clients for ${RUN_TIME:-ever}"
 if [ "$SILENT" -eq 1 ]
 then

load-gen/load-gen.sh

@@ -15,6 +15,9 @@ HOST="http://localhost:8080"
 # Error flag
 ERROR=0
 
+# Verbose mode
+LOAD_DEBUG=1
+
 # Daemon flag
 DAEMON="-it"
 SILENT=0
@@ -24,7 +27,8 @@ USAGE="\
 loadgen.sh
 
 e - error flag
-d - run in background
+v - verbose mode. It implies the setting of LOAD_DEBUG and of error flag
+d - run in background. It implies the setting of SILENT mode, so the stdout will not be floaded. If -v is used, the load debug will still be printed in the stdout, even if the stdout is suppressed.
 n - number of clients
 t - time to run n clients
 h - target host
@@ -42,12 +46,16 @@ eval $(egrep '[A-Z]+=' ../.env)
 echo "Repo $REPO"
 echo "Tag $TAG"
 
-while getopts 'edn:t:h:' OPT
+while getopts 'edvn:t:h:' OPT
 do
     case $OPT in
         e)
             ERROR=1
             ;;
+        v)
+            LOAD_DEBUG=1
+            ERROR=1
+            ;;
         d)
             DAEMON="-d"
             SILENT=1
@@ -92,9 +100,14 @@ do
     esac
 done
 
+rm -rf logs/*
 docker run \
     $DAEMON \
     --name loadgen \
+    --volume ${PWD}/robot-shop.py:/load/robot-shop.py \
+    --volume ${PWD}/entrypoint.sh:/load/entrypoint.sh \
+    --volume ${PWD}/utilities:/load/utilities \
+    --volume ${PWD}/logs:/load/logs \
     --rm \
     --network=host \
     -e "HOST=$HOST" \
@@ -102,5 +115,6 @@ docker run \
     -e "RUN_TIME=$RUN_TIME" \
     -e "SILENT=$SILENT" \
     -e "ERROR=$ERROR" \
+    -e "LOAD_DEBUG=$LOAD_DEBUG" \
     ${REPO}/rs-load:${TAG}
 

load-gen/robot-shop.py

@@ -2,8 +2,11 @@ import os
 import random
 
 from locust import HttpUser, task, between
+from utilities.CSVWriter import CSVWriter
 from random import choice
 from random import randint
+from sys import argv
+from datetime import date
 
 class UserBehavior(HttpUser):
     wait_time = between(2, 10)
@@ -26,9 +29,26 @@ class UserBehavior(HttpUser):
         "60.242.161.215"
     ]
 
+    phplog = None
+    php_services_api_prefix = '/api/ratings/api'
+    php_service_rate = '/rate'
+    php_service_fetch = '/fetch'
+    php_fieldnames = ['REQTYPE', 'SERVICE', 'INPUT', 'HEADER']
+    my_csv_writer = None
+
     def on_start(self):
         """ on_start is called when a Locust start before any task is scheduled """
         print('Starting')
+        print('LOAD_DEBUG: ', os.environ.get("LOAD_DEBUG"))
+
+        if os.environ.get('LOAD_DEBUG') == '1':
+            print("ARGS ARE:\n\"")
+            print("\n".join(argv))
+            print('End of ARGS printing\n')
+            print('on start. php_fieldnames:{}', format(self.php_fieldnames))
+
+        self.my_csv_writer = CSVWriter("logs/php_services_calls.csv", self.php_fieldnames)
+        self.phplog = open("logs/phplog.txt", "a")
 
     @task
     def login(self):
@@ -44,6 +64,7 @@ class UserBehavior(HttpUser):
 
     @task
     def load(self):
+        self.phplog.write('new user, new load task\n')
         fake_ip = random.choice(self.fake_ip_addresses)
 
         self.client.get('/', headers={'x-forwarded-for': fake_ip})
@@ -61,12 +82,31 @@ class UserBehavior(HttpUser):
                 if item['instock'] != 0:
                     break
 
+            self.phplog.write('fake_ip: {}\n'.format(fake_ip))
+
+            headers={'x-forwarded-for': fake_ip}
             # vote for item
             if randint(1, 10) <= 3:
-                self.client.put('/api/ratings/api/rate/{}/{}'.format(item['sku'], randint(1, 5)), headers={'x-forwarded-for': fake_ip})
+                ratevalue = randint(1, 5)
+                put_rate_api_str = '{}{}/{}/{}'.format(self.php_services_api_prefix, self.php_service_rate, item['sku'], ratevalue )
+                if os.environ.get('LOAD_DEBUG') == '1':
+                    self.phplog.write('ratevalue: {}\n'.format(ratevalue))
+                    self.phplog.write('item: {}\n'.format(item['sku']))
+                    self.phplog.write('put_rate_api_str: {}\n'.format(put_rate_api_str))
+                    self.phplog.flush()
+                self.my_csv_writer.writerow({'REQTYPE': 'PUT', 'SERVICE': '{}'.format(self.php_service_rate), 'INPUT': '{}/{}'.format(item['sku'], ratevalue ), 'HEADER': '{}'.format(headers)})
+                self.client.put(put_rate_api_str, headers)
 
             self.client.get('/api/catalogue/product/{}'.format(item['sku']), headers={'x-forwarded-for': fake_ip})
-            self.client.get('/api/ratings/api/fetch/{}'.format(item['sku']), headers={'x-forwarded-for': fake_ip})
+
+            get_rate_api_str = '{}{}/{}'.format(self.php_services_api_prefix, self.php_service_fetch, item['sku'])
+            if os.environ.get('LOAD_DEBUG') == '1':
+                self.phplog.write('item: {}\n'.format(item['sku']))
+                self.phplog.write('get_rate_api_str: {}\n'.format(get_rate_api_str))
+                self.phplog.flush()
+            self.my_csv_writer.writerow({'REQTYPE': 'GET', 'SERVICE': '{}'.format(self.php_service_fetch), 'INPUT': '{}'.format(item['sku']), 'HEADER': '{}'.format(headers) })
+            self.client.get(get_rate_api_str, headers={'x-forwarded-for': fake_ip})
+
             self.client.get('/api/cart/add/{}/{}/1'.format(uniqueid, item['sku']), headers={'x-forwarded-for': fake_ip})
 
         cart = self.client.get('/api/cart/cart/{}'.format(uniqueid), headers={'x-forwarded-for': fake_ip}).json()

load-gen/utilities/CSVReader.py

@@ -0,0 +1,19 @@
+import csv
+
+# Refs:
+# [Tutorial1] https://rharshad.com/locust-load-test/
+class CSVReader:
+    def __init__(self, filepath):
+        try:
+            file = open(filepath)
+        except TypeError:
+            pass
+        self.file = file
+        self.reader = csv.reader(file)
+
+    def __next__(self):
+        try:
+            return next(self.reader)
+        except StopIteration:
+            self.file.seek(0, 0)
+            return next(self.reader)

load-gen/utilities/CSVWriter.py

@@ -0,0 +1,45 @@
+import os
+import csv
+from datetime import date
+
+class CSVWriter:
+    def __init__(self, filepath, custom_fields=[]):
+        if os.environ.get('LOAD_DEBUG') == '1':
+            print('CSVWriter init/args: filepath:\'{}\' custom_fields\'{}\''.format(filepath,custom_fields))
+
+        try:
+            file = open(filepath,'a')
+        except TypeError as typeErr:
+            print(f"CSVWriter init/Unexpected {err=}, {type(typeErr)=}")
+            raise
+
+        print('CSVWriter init/pass')
+        self.file = file
+
+        if not custom_fields:
+            print('CSVWriter init/empty params')
+        else:
+            self.fieldnames = custom_fields
+
+        try:
+            self.writer = csv.DictWriter(self.file, self.fieldnames)
+            self.writer.writeheader()
+        except BaseException as err:
+            print(f"CSVWriter init/Unexpected {err=}, {type(err)=}")
+            raise
+
+        self.file.flush()
+
+    def writerow(self, row: dict):
+        if os.environ.get('LOAD_DEBUG') == '1':
+            print('CSVWriter/writerow/args: {}',format(row))
+
+        if not row:
+            print('CSVWriter/writerow/empty arg')
+        else:
+            try:
+                self.writer.writerow(row)
+                self.file.flush()
+            except BaseException as err:
+                print(f"CSVWriter/writerow/Unexpected {err=}, {type(err)=}")
+                raise