diff --git a/.gitignore b/.gitignore
index ff7cba4bdfb5c5181bf7433d6538705c834bd5d7..a106668eb7a436f9ced76c4f626300a92cc0a3de 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
plots/
-plot_general/plot.pdf
+plots_kpi/
+*/plot.pdf
# Ignore all intermediate data that can pile up in local repositories
data/*/data_*.csv
@@ -8,6 +9,9 @@ data/*/*.png
data/*/*.pdf
202[0-9]*/*.pdf
202[0-9]*/*.png
+fit.log
+susage.csv
+_OVERALL_USAGE.csv
# Ignore all data subdirectory plots except those explicitly added
data/*/filter.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f484698a4cdc24414dbf0f7bab24f2cd80ba3be6..e68db412d630777328f9be1fd4fe10566be566f2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,11 +28,15 @@ plot_general:
GIT_SUBMODULE_STRATEGY: recursive
script:
- mkdir -v plots
- - cd plot_general
+ - cd plots_scripts
- bash ./plot.sh
- bash ./plot_replotgraphs.sh
+ - cd ../plots_kpi_scripts
+ - bash ./plot.sh
- cd ..
- pdfunite plots/pdf/*.pdf plots/all.pdf
+ - pdfunite plots_kpi/pdf/*.pdf plots_kpi/all.pdf
artifacts:
paths:
- plots
+ - plots_kpi
diff --git a/README.md b/README.md
index 22a0af68616223a523eba16d88d5469905d154b4..eaabce5161c7537896b5b47b5839b6d1bf8d12d3 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,10 @@ The data is incorporated via submodules from separate projects.
## Generated Plots (most recent)
-The plots are generated automatically when changes are detected or the generation pipeline is triggered.
+All plots are generated automatically when changes are detected or the generation pipeline is triggered.
+Also daily plots are generated.
+
+### General overview plots
* [**Latest build, all graphs**](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/browse/plots?job=plot_general)
* [Download all graphs in zip file](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/download/plots?job=plot_general)
@@ -14,20 +17,15 @@ The plots are generated automatically when changes are detected or the generatio
* [Gallery view](gallery/gallery.md)
* [Graphs pdf collection](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/all.pdf?job=plot_general)
* Example of inline svg (AAI users):
- 
-
-
-## Services
+ 
-This is the list of services currently being considered.
-
-* Helmholtz AAI
-* Helmholtz Backbone (VPN)
-* Helmholtz Codebase - (Gitlab at HZDR)
-* Nubes (HZB)
-* Mattermost (HZDR)
-* Openstack (FZJ)
-* bwSync&Share (KIT)
+### KPI plots
+* [**Latest build, all graphs**](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/browse/plots_kpi?job=plot_general)
+* [Download all graphs in zip file](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/download/plots_kpi?job=plot_general)
+* [Previously used graphs](example_graphs.md), used in reports, reproduced with most recent data.
+* [Graphs pdf collection](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots_kpi/all.pdf?job=plot_general)
+* Example of KPI plot (HZDR Codebase):
+ 
## Local execution
@@ -45,7 +43,7 @@ git submodule update --remote --recursive
Necessary for converting and plotting: `gnuplot`, `gawk`, `imagemagick`, `poppler-utils`.
```code
-cd plot_general # or any other of the sub-folders
+cd plots_scripts # or any other of the sub-folders
./plot.sh
```
diff --git a/data/desy_events/T_onboard.txt b/data/desy_events/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..47ab2984088b773b8978104a3b2416c98f1b4220
--- /dev/null
+++ b/data/desy_events/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_events.txt
\ No newline at end of file
diff --git a/data/desy_notes_15min/T_onboard.txt b/data/desy_notes_15min/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..0e71a82543eccc0619369c8e6cf1359bc6744cb6
--- /dev/null
+++ b/data/desy_notes_15min/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_notes.txt
\ No newline at end of file
diff --git a/data/desy_notes_daily/T_onboard.txt b/data/desy_notes_daily/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..0e71a82543eccc0619369c8e6cf1359bc6744cb6
--- /dev/null
+++ b/data/desy_notes_daily/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_notes.txt
\ No newline at end of file
diff --git a/data/desy_notes_weekly/T_onboard.txt b/data/desy_notes_weekly/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..0e71a82543eccc0619369c8e6cf1359bc6744cb6
--- /dev/null
+++ b/data/desy_notes_weekly/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_notes.txt
\ No newline at end of file
diff --git a/data/desy_syncnshare_consumption/T_onboard.txt b/data/desy_syncnshare_consumption/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..27cc78a6de96f059372a7781aaf55fb3b1e12bfa
--- /dev/null
+++ b/data/desy_syncnshare_consumption/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_syncnshare.txt
\ No newline at end of file
diff --git a/data/desy_syncnshare_users_aai/T_onboard.txt b/data/desy_syncnshare_users_aai/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..27cc78a6de96f059372a7781aaf55fb3b1e12bfa
--- /dev/null
+++ b/data/desy_syncnshare_users_aai/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_syncnshare.txt
\ No newline at end of file
diff --git a/data/desy_syncnshare_users/data.csv b/data/desy_syncnshare_users_aai/data.csv
similarity index 100%
rename from data/desy_syncnshare_users/data.csv
rename to data/desy_syncnshare_users_aai/data.csv
diff --git a/data/desy_syncnshare_users_all/T_onboard.txt b/data/desy_syncnshare_users_all/T_onboard.txt
new file mode 120000
index 0000000000000000000000000000000000000000..27cc78a6de96f059372a7781aaf55fb3b1e12bfa
--- /dev/null
+++ b/data/desy_syncnshare_users_all/T_onboard.txt
@@ -0,0 +1 @@
+../../metadata/T_onboard/desy_syncnshare.txt
\ No newline at end of file
diff --git a/data/desy_syncnshare_users_all/data.csv b/data/desy_syncnshare_users_all/data.csv
new file mode 120000
index 0000000000000000000000000000000000000000..5f1f01f5372c747806b0d86715aa4635ec5694c8
--- /dev/null
+++ b/data/desy_syncnshare_users_all/data.csv
@@ -0,0 +1 @@
+../../subprojects/DESY-syncnshare/stats/users_all.csv
\ No newline at end of file
diff --git a/data/kpi-weights.csv b/data/kpi-weights.csv
new file mode 100644
index 0000000000000000000000000000000000000000..469f17fc55bdb78e6e5b57fe7c19fc24c8c7bd6b
--- /dev/null
+++ b/data/kpi-weights.csv
@@ -0,0 +1,67 @@
+service_name,col_number,kpi,weight
+aai_active_users,2,Active User [number of active user],0
+aai_users,2,registered users,0
+desy_notes_15min,2,onlineNotes,0
+desy_notes_15min,3,onlineUsers,0
+desy_notes_15min,4,notesCount,0
+desy_notes_15min,5,registeredUsers,0
+desy_notes_15min,6,onlineRegisteredUsers,0
+desy_notes_daily,2,peak onlineNotes,0
+desy_notes_daily,3,peak onlineUsers,0
+desy_notes_daily,4,peak notesCount,0
+desy_notes_daily,5,peak registeredUsers,0
+desy_notes_daily,6,peak onlineRegisteredUsers,0
+desy_notes_weekly,2,peak onlineNotes,0
+desy_notes_weekly,3,peak onlineUsers,0
+desy_notes_weekly,4,peak notesCount,0
+desy_notes_weekly,5,peak registeredUsers,0
+desy_notes_weekly,6,peak onlineRegisteredUsers,0
+desy_syncnshare_users_aai,2,Users authenticated via Helmholtz AAI,0.5
+desy_syncnshare_users_all,2,All Users,0.5
+fzj_b2share,2,Disk Space [Byte],0
+fzj_b2share,3,Records [Number of records],0
+fzj_b2share,4,Files [Number of files],0
+fzj_jupyter,2,successful jobs,0.16667
+fzj_jupyter,3,unique users,0.5
+fzj_jupyter,4,nodes,0.16667
+fzj_jupyter,5,gpus,0.16667
+fzj_openstack,2,CPU Hours [h],0.2
+fzj_openstack,3,RAM MB-Hours [MB h],0.2
+fzj_openstack,4,Disk GB-Hours [GB h],0.2
+fzj_openstack,5,Servers,0.2
+fzj_openstack,6,Projects,0.2
+helmholtz-cloud-services,2,services [number],0
+hzb_nubes,2,active user,0.5
+hzb_nubes,3,folders,0
+hzb_nubes,4,files,0.125
+hzb_nubes,5,user-storage [G],0.125
+hzb_nubes,6,groupfolders – on top,0
+hzb_nubes,7,groupfolders,0.125
+hzb_nubes,8,Groupfolder-files,0
+hzb_nubes,9,group-storage [G],0
+hzb_nubes,10,shares,0.125
+hzdr_codebase_active_users,2,number_of_users,0.25
+hzdr_codebase_active_users,3,last_active_1_days,0
+hzdr_codebase_active_users,4,last_active_7_days,0.25
+hzdr_codebase_active_users,5,last_active_14_days,0
+hzdr_codebase_active_users,6,last_active_30_days,0
+hzdr_codebase_active_users,7,last_active_60_days,0
+hzdr_codebase_project_statistics,2,number_of_projects,0.125
+hzdr_codebase_project_statistics,3,total_commit_count,0.125
+hzdr_codebase_project_statistics,4,total_storage_size,0.125
+hzdr_codebase_project_statistics,5,total_repository_size,0
+hzdr_codebase_project_statistics,6,total_wiki_size,0
+hzdr_codebase_project_statistics,7,total_lfs_object_size,0
+hzdr_codebase_project_statistics,8,total_job_artifacts_size,0
+hzdr_codebase_project_statistics,9,last_active_1_days,0
+hzdr_codebase_project_statistics,10,last_active_7_days,0.125
+hzdr_codebase_project_statistics,11,last_active_14_days,0
+hzdr_codebase_project_statistics,12,last_active_30_days,0
+hzdr_codebase_project_statistics,13,last_active_60_days,0
+hzdr_helpdesk,2,number_of_created_tickets_per_7_days,1
+hzdr_mattermost,2, post count ,0.25
+hzdr_mattermost,3, team count ,0.25
+hzdr_mattermost,4, daily active users ,0
+hzdr_mattermost,5, monthly active users,0.5
+kit_bwsyncnshare,2,nc-serverinfo-api_nc-si.mean,0
+vpn_centres,2,vpn_centres [centres],0
diff --git a/gallery/gallery.md b/gallery/gallery.md
index db4f3b92f6b0c166a52d8e9e4f8ab53afb239ea0..29ed902bdfdbb1c858f3fa7f7af8f46eff69675f 100644
--- a/gallery/gallery.md
+++ b/gallery/gallery.md
@@ -89,8 +89,11 @@
##### desy_syncnshare_consumption-plot_4.svg
-##### desy_syncnshare_users-plot_2.svg
-
+##### desy_syncnshare_users_aai-plot_2.svg
+
+
+##### desy_syncnshare_users_all-plot_2.svg
+
##### fzj_b2share-plot_2.svg
diff --git a/metadata/T_onboard/add_metadata.sh b/metadata/T_onboard/add_metadata.sh
new file mode 100755
index 0000000000000000000000000000000000000000..d2ca74ff96cfab7e995aeea5ae38de30a3699993
--- /dev/null
+++ b/metadata/T_onboard/add_metadata.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# for every text file in here, depicting one service onboarding time each....
+for i in *.txt; do
+
+ # ...we look for corresponding data files, with arbitary sub-KPI naming
+ for j in ../../data/"${i%.txt}"*; do
+
+ # if this is really a directory link data.
+ if [ -d "$j" ]; then
+ ln -sv ../../metadata/T_onboard/"$i" "$j"/T_onboard.txt
+ fi
+ done
+done
\ No newline at end of file
diff --git a/metadata/T_onboard/desy_events.txt b/metadata/T_onboard/desy_events.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ef348ad87105953c3be8e193ea2f2de01e9d3d7d
--- /dev/null
+++ b/metadata/T_onboard/desy_events.txt
@@ -0,0 +1 @@
+T_onboard="2021-12-08" # according to https://gitlab.hzdr.de/hifis/cloud/access-layer/portal/-/tree/b27072f865a84f270cd23942c31059df4037ae98/cerebrum/data/marketService
\ No newline at end of file
diff --git a/metadata/T_onboard/desy_notes.txt b/metadata/T_onboard/desy_notes.txt
new file mode 100644
index 0000000000000000000000000000000000000000..417751f5e51cd2a00e16c59893f1578e48aaa420
--- /dev/null
+++ b/metadata/T_onboard/desy_notes.txt
@@ -0,0 +1 @@
+T_onboard="2021-12-10" # according to https://gitlab.hzdr.de/hifis/cloud/access-layer/portal/-/tree/a40dc87391130c84aabeeed9dc3971c30193c63c/cerebrum/data/marketService
\ No newline at end of file
diff --git a/metadata/T_onboard/desy_syncnshare.txt b/metadata/T_onboard/desy_syncnshare.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e858e6b6c7fdb9f2e9054117e8b84c192ad398d6
--- /dev/null
+++ b/metadata/T_onboard/desy_syncnshare.txt
@@ -0,0 +1 @@
+T_onboard="2021-09-07" # according to https://gitlab.hzdr.de/hifis/cloud/access-layer/portal/-/blob/9e0283dad6987cc3dbabbb78d79e6cb0472b772b/cerebrum/data/marketService.json
\ No newline at end of file
diff --git a/plots_kpi_scripts/filter.sh b/plots_kpi_scripts/filter.sh
new file mode 120000
index 0000000000000000000000000000000000000000..c46984c123c45f09791a5c8236b615274462cb24
--- /dev/null
+++ b/plots_kpi_scripts/filter.sh
@@ -0,0 +1 @@
+../plots_scripts/filter.sh
\ No newline at end of file
diff --git a/plots_kpi_scripts/plot.plt b/plots_kpi_scripts/plot.plt
new file mode 100755
index 0000000000000000000000000000000000000000..0576a8246d6418210392be33eb9e262481b39993
--- /dev/null
+++ b/plots_kpi_scripts/plot.plt
@@ -0,0 +1,174 @@
+#!/bin/gnuplot
+
+# Check if the separator is a "," or ";". We want to have at least two columns, so we'll test:
+sep=","
+set datafile separator sep
+stats 'data_plot.csv' nooutput
+# If we detected too few columns, try another separator.
+if (STATS_columns<=1) {
+ sep=";"
+ set datafile separator sep
+ stats 'data_plot.csv' nooutput
+}
+
+print "Detected ".STATS_columns." columns."
+
+if (STATS_columns<=1) {
+ print "Detected too few columns. Exiting."
+ exit
+}
+
+set yrange [0:*]
+set xdata time
+set format x "%Y-%m-%d"
+
+# There are chaotically many ways to encode date/time so far. We presume a standard format, but allow to override the format via timefmt.txt
+set timefmt "%Y-%m-%d %H:%M"
+if (system("[ ! -f timefmt.txt ]; echo $?")) {
+ load "timefmt.txt"
+}
+
+set xrange ["2020-01-01":"2022-04-02"]
+set xtics rotate by 45
+set xtics textcolor rgbcolor "black" offset -4,-3
+
+set xtics '2020-01-01 00:00',365*24*3600
+set mxtics 12
+set grid xtics nomxtics ytics
+
+set key top left
+set linetype 2 dashtype 2
+
+
+# set some coordinates for diagram formatting. Can also be used to draw multiple aligned diagrams in one canvas, if wanted (multiplot)
+MP_LEFT = .1
+MP_RIGHT = .95
+MP_BOTTOM = .14
+MP_TOP = .93
+MP_xGAP = 0.1
+MP_yGAP = 0.02
+
+
+# fitting params
+set fit errorscaling errorvariables
+FIT_LIMIT=1e-15 # this should help to fit even very degenerate plots
+FIT_MAXITER=1000
+
+# fitting function
+susage(x) = m*x + y0
+
+# Plot all columns: One file for each column.
+
+filename='data_plot.csv'
+# since replot does not work reliably, we repeat the whole plot sequence
+# explicitly for every output format wanted.
+
+# log to one file per service
+set print "susage.csv"
+
+do for [i=2:STATS_columns] {
+
+ # starting parameters for each fit
+ y0=100
+ m=1
+
+ # fit slope for data points at least from 2020 (to exclude any possible data points from 2019 or before)
+ # for annual reporting 2021: AND fit only for data that is <=2021
+
+ fmt = "%Y-%m-%d"
+ set xdata time
+ set timefmt fmt
+
+ # reporting date (end of measurement period)
+ T_report="2021-12-31" # hard-coded for every report
+
+ # starting dates.
+ T_logstart=system("cat ".filename." | tr -d '\r' | awk 'BEGIN{FS=\"".sep."\"}{if (NR>1 && NF>1) {print $1; exit;}}'")
+
+
+ # Onboarding date can be no earlier than the start of the pilot Helmholtz Cloud, i.e. 2021-03-29.
+ # https://hifis.net/news/2021/03/29/helmholtz-cloud-beta
+
+ T_onboard="2021-03-29"
+ # if there is a specific file determining another T_onboard for that service, defined manually, use that instead:
+ if (system("[ ! -f T_onboard.txt ]; echo $?")) {
+ load "T_onboard.txt"
+ }
+
+ if (strptime(fmt, T_onboard) > strptime(fmt, T_logstart)) {
+ T_fitstart_overall=T_onboard
+ } else {
+ T_fitstart_overall=T_logstart
+ }
+
+ # Do the fitting
+ fit [strptime(fmt, T_fitstart_overall):strptime(fmt, T_report)] susage(x) filename u 1:i via m,y0
+
+
+ # Later: Yearly procedure for yearly reports, not defined for 2021 annual report yet.
+ # Will take care of this later for reporting 2023 ff.
+ # But we need a placeholder here.
+ T_fitstart_yearly="N/A"
+ T_reference_yearly="N/A"
+
+ # Now we have a fit and can decide where to put the T_reference_overall.
+ # Per default, T_reference_overall should be equal to T_fitstart_overall.
+ # Exceptions MAY be that the KPI at this date is very low (close to zero), such that a division by this KPI makes no sense.
+ # In that case, a separate date MAY be defined by Cloud Cluster Management in T_reference_overall.txt.
+ T_reference_overall=T_fitstart_overall
+ if (system("[ ! -f T_reference_overall.txt ]; echo $?")) {
+ load "T_reference_overall.txt"
+ }
+
+ # Using T_reference_overall and T_report, the corresponding KPIs at these dates are calculated from the fit.
+ KPI_e=susage(strptime(fmt, T_report))
+ KPI_a=susage(strptime(fmt, T_reference_overall))
+
+ # Compute the raw dollar value
+ kpi_raw=KPI_e/KPI_a-1
+
+ # check if we actually want to consider this value (service was on board long enough, i.e. >=90 days)
+ if ((strptime(fmt, T_report)-strptime(fmt, T_fitstart_overall)) >= (90.*3600.*24.)) {
+ kpi_raw_title=sprintf("%.3g - subject to weighting.", kpi_raw);
+ } else {
+ kpi_raw_title=sprintf("%.3g - NOT CONSIDERED: less than 90 days data available.", kpi_raw);
+ kpi_raw=0;
+ }
+
+
+ # Extract KPI name. Seemingly, the header name of the column cannot be directly derived with gnuplot,
+ # instead we have to circle via shell and pipes (urgh)
+ kpi_name=system("head -n 1 ".filename." | tr -d '\r' | awk 'BEGIN{FS=\"".sep."\"}{print $".i."}'")
+
+ # Printing to table
+ logdata=sprintf("%s,%d,%s,%g,%g,%g,%g,,%s,%s,%s,%s,%s,%s,%s,%g,%g,%g\n",servicename,i,kpi_name,m,m_err,y0,y0_err,T_logstart,T_onboard,T_fitstart_overall,T_reference_overall,T_fitstart_yearly,T_reference_yearly,T_report,KPI_a,KPI_e,kpi_raw)
+ print logdata
+
+ # Plotting (once to pdf, once to svg)
+ do for [IDX = 0:1] {
+ if (IDX==0) {
+ set terminal pdf color noenhanced size 7,5 dashed
+ set output 'plot_'.i.'.pdf'
+ } else {
+ set terminal svg noenhanced size 800,600
+ set output 'plot_'.i.'.svg'
+ }
+
+ set multiplot layout 1,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
+ set title servicename
+
+ # print linear fitting variables and errors
+ set label sprintf("m=%.2g +- %.3g (%.3g %%),\ny0=%.2g +- %.3g (%.3g %%)\nT_reference_overall=%s, T_report=%s\nkpi-raw=%s",m,m_err,m_err/m*100,y0,y0_err,y0_err/y0*100,T_reference_overall,T_report,kpi_raw_title) left at graph 0.1, graph 0.7
+
+ plot filename u 1:i w lp pt 7 ps 0.25 lc black title columnheader ,\
+ (x>=strptime(fmt, T_fitstart_overall) && x<= strptime(fmt, T_report)) ? susage(x) : 1/0 w l linewidth 4 lc "#55bb55" notitle ,\
+ '+' using (T_onboard):(susage(strptime(fmt, T_onboard))) w p pt 8 ps 1.5 lw 3 lc "#000000" title 'T_onboard' ,\
+ '+' using (T_logstart):(susage(strptime(fmt, T_logstart))) w p pt 2 ps 1.5 lw 3 lc "#888888" title 'T_logstart' ,\
+ '+' using (T_fitstart_overall):(susage(strptime(fmt, T_fitstart_overall))) w p pt 6 ps 1.5 lw 3 lc "#00bb00" title 'T_fitstart_overall' ,\
+ '+' using (T_reference_overall):(susage(strptime(fmt, T_reference_overall))) w p pt 4 ps 1.5 lw 3 lc "#0000bb" title 'T_reference_overall' ,\
+ '+' using (T_report):(susage(strptime(fmt, T_report))) w p pt 10 ps 1.5 lw 3 lc "#ff0000" title 'T_report'
+
+ unset multiplot
+ unset label
+ }
+}
diff --git a/plots_kpi_scripts/plot.sh b/plots_kpi_scripts/plot.sh
new file mode 100755
index 0000000000000000000000000000000000000000..2751a09b9dc67871de65cd0348d98f0fee80f977
--- /dev/null
+++ b/plots_kpi_scripts/plot.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+if [ ! -d ../plots_kpi/pdf ]; then mkdir -pv ../plots_kpi/pdf; fi
+if [ ! -d ../plots_kpi/svg ]; then mkdir -pv ../plots_kpi/svg; fi
+if [ ! -d ../plots_kpi/log ]; then mkdir -pv ../plots_kpi/log; fi
+
+
+# create plot(s) for every data.csv found within data folder
+for datafile in ../data/*/data.csv; do
+{
+ echo -e "\n------------------- ${datafile}"
+ test -f ${datafile} || {
+ echo "File ${datafile} does not exist, skipping"
+ continue
+ }
+ # cut on last "/" to get folder name
+ folder=$(echo $datafile | rev | cut -d "/" -f 2- | rev)
+ folder=`dirname ${datafile}`
+ basefile=`basename ${datafile}`
+
+ # Clean up any remaining *_plot.csv if there are any. They must be created on the fly.
+ if [ -e "${datafile%.csv}_plot.csv" ]; then
+ rm -v "${datafile%.csv}_plot.csv"
+ fi
+
+ # If there is a filter.sh, use it.
+ # Otherwise, link the standard filter.sh and use this.
+ if [ ! -e "$folder/"filter.sh ] || [ -h "$folder/"filter.sh ]; then
+ # remove the non-existing file, just to eliminate broken symlinks
+ rm -f "$folder/"filter.sh
+ echo ln -sv "$(pwd)"/filter.sh "$folder"/;
+ ln -sv "$(pwd)"/filter.sh "$folder"/;
+ fi
+ pushd "$folder"
+ ./filter.sh
+ popd
+ echo "Filter done"
+
+ # Plotting.
+ # Either
+ # - we can use the standard plot.plt and link it there, or
+ # - there is a specific plot.plt already deposited for the specific folder.
+
+ if [ ! -e "$folder"/plot.plt ] || [ -h "$folder"/plot.plt ]; then
+ # remove the non-existing file, just to eliminate broken symlinks
+ rm -f "$folder"/plot.plt
+ echo ln -sv "$(pwd)"/plot.plt "$folder"/;
+ ln -sv "$(pwd)"/plot.plt "$folder"/;
+ fi
+
+ pushd "$folder"
+ if [ -e fit.log ]; then rm -v fit.log; fi
+ gnuplot -e "servicename='`basename ${folder}`'" plot.plt
+ popd
+
+ # Convert files as necessary, move plot files to artifacts folder, concatenate name.
+ for plotfile in "$folder"/plot*.pdf; do
+ {
+ if [ -e "$plotfile" ]; then
+ pdffile=$(echo $plotfile | cut -d "/" -f 3- | sed 's/\//-/g')
+ mv -v "$plotfile" "$pdffile"
+ mv -v "${plotfile%.pdf}.svg" "${pdffile%.pdf}.svg"
+
+ # Move all files types to respective folders
+ mv -v "$pdffile" ../plots_kpi/pdf/
+ mv -v "${pdffile%.pdf}.svg" ../plots_kpi/svg/
+
+ mv -v "$folder"/fit.log ../plots_kpi/log/"${pdffile%.pdf}.fit.log"
+ mv -v "$folder"/susage.csv ../plots_kpi/log/"${pdffile%.pdf}.susage.csv"
+ fi
+ }
+ done
+}
+done
+
+# collect dollars, add kpi name in first column, skip empty lines
+pushd ../plots_kpi/log
+{
+ echo "service_name,col_number,kpi,m,m_err,y0,y0_err,,T_logstart,T_onboard,T_fitstart_overall,T_reference_overall,T_fitstart_yearly,T_reference_yearly,T_e,KPI_a,KPI_e,dollar-raw"
+ for i in *.susage.csv; do
+ if [ -e "$i" ]; then
+ gawk -v fname=${i%.susage.csv} 'BEGIN{FS=","}{
+ # exclude empty lines, lines with no data
+ if (NF>1) printf("%s\n",$0)
+ }' "$i"
+ fi
+ done
+} > ../_OVERALL_USAGE-raw.csv
+
+# weight KPI
+cd ..
+gawk 'BEGIN{FS=","; file_id=0;}{
+ if (FNR<FNR_old) file_id++;
+ FNR_old=FNR
+
+ if (file_id==0) {
+ # when reading the index file:
+ # read all weights into corresponding indexed field.
+ idx=$1"-"$2
+ weights[idx]=$4;
+ } else if (FNR==1) {
+ # when reading data file:
+ # first line: plot title line plus additional columns
+ printf ("%s,weight,weighted KPI\n",$0);
+ } else {
+ # when reading data file beloow title line.
+ # print everything and additional weighted computation.
+ idx=$1"-"$2
+ weighted_kpi=$18*weights[idx]
+ printf ("%s,%g,%g\n", $0, weights[idx], weighted_kpi);
+
+ KPI_overall+=weighted_kpi;
+ }
+
+}END{
+ printf("%g\n", KPI_overall) > "overall-kpi.csv"
+}' "../data/kpi-weights.csv" "_OVERALL_USAGE-raw.csv" | tee _OVERALL_USAGE.csv
+popd
+
+# plot summary of weighted KPI
+gnuplot plot_overall_kpi.plt
\ No newline at end of file
diff --git a/plots_kpi_scripts/plot_overall_kpi.plt b/plots_kpi_scripts/plot_overall_kpi.plt
new file mode 100755
index 0000000000000000000000000000000000000000..c0424192986f56f57955b318866fa0be5c7e7301
--- /dev/null
+++ b/plots_kpi_scripts/plot_overall_kpi.plt
@@ -0,0 +1,41 @@
+#!/bin/gnuplot
+
+# Check if the separator is a "," or ";". We want to have at least two columns, so we'll test:
+sep=","
+set datafile separator sep
+
+fn="../plots_kpi/_OVERALL_USAGE.csv"
+
+set ylabel "Weighted KPI"
+
+MP_LEFT = .03
+MP_RIGHT = .97
+MP_BOTTOM = .35
+MP_TOP = .93
+MP_xGAP = 0.1
+MP_yGAP = 0.02
+
+set xtics rotate by 90
+set xtics textcolor rgbcolor "black" offset 0,-12
+
+set style fill solid 1.00
+set style histogram clustered gap 1
+
+set grid
+
+
+# Plotting (once to pdf, once to svg)
+do for [IDX = 0:1] {
+ if (IDX==0) {
+ set terminal pdf color noenhanced size 20,7 dashed
+ set output '../plots_kpi/_OVERALL_USAGE_overview.pdf'
+ } else {
+ set terminal svg noenhanced size 1500,750
+ set output '../plots_kpi/_OVERALL_USAGE_overview.svg'
+ }
+
+
+ set multiplot layout 1,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
+ plot fn u ($20>0?$20:1/0):xtic(1) with histogram notitle
+ unset multiplot
+}
diff --git a/plot_general/filter.sh b/plots_scripts/filter.sh
similarity index 100%
rename from plot_general/filter.sh
rename to plots_scripts/filter.sh
diff --git a/plot_general/plot.plt b/plots_scripts/plot.plt
similarity index 100%
rename from plot_general/plot.plt
rename to plots_scripts/plot.plt
diff --git a/plot_general/plot.sh b/plots_scripts/plot.sh
similarity index 100%
rename from plot_general/plot.sh
rename to plots_scripts/plot.sh
diff --git a/plot_general/plot_replotgraphs.sh b/plots_scripts/plot_replotgraphs.sh
similarity index 100%
rename from plot_general/plot_replotgraphs.sh
rename to plots_scripts/plot_replotgraphs.sh
diff --git a/subprojects/DESY-notes b/subprojects/DESY-notes
index b0b159597a98f32c59241a5163774901229acb2e..c73aebff08032b9d00002616ce827a1a94d71f31 160000
--- a/subprojects/DESY-notes
+++ b/subprojects/DESY-notes
@@ -1 +1 @@
-Subproject commit b0b159597a98f32c59241a5163774901229acb2e
+Subproject commit c73aebff08032b9d00002616ce827a1a94d71f31
diff --git a/subprojects/DESY-syncnshare b/subprojects/DESY-syncnshare
index b743df3ca889c7b94af1f3cc5441276cd8fce3bd..f1be304b843075f4210ad2f182f9e8b7108d6295 160000
--- a/subprojects/DESY-syncnshare
+++ b/subprojects/DESY-syncnshare
@@ -1 +1 @@
-Subproject commit b743df3ca889c7b94af1f3cc5441276cd8fce3bd
+Subproject commit f1be304b843075f4210ad2f182f9e8b7108d6295
diff --git a/subprojects/KIT-bwsns b/subprojects/KIT-bwsns
index b4cc3cdd4273655c9b04e73eee889b436cea8bc7..35f51ed2f277f10724ce51fe9e5d017b5f7cb64f 160000
--- a/subprojects/KIT-bwsns
+++ b/subprojects/KIT-bwsns
@@ -1 +1 @@
-Subproject commit b4cc3cdd4273655c9b04e73eee889b436cea8bc7
+Subproject commit 35f51ed2f277f10724ce51fe9e5d017b5f7cb64f