Service usage 2020 21

.gitignore

 plots/
-plot_general/plot.pdf
+plots_kpi/
+*/plot.pdf
 
 # Ignore all intermediate data that can pile up in local repositories
 data/*/data_*.csv
@@ -8,6 +9,9 @@ data/*/*.png
 data/*/*.pdf
 202[0-9]*/*.pdf
 202[0-9]*/*.png
+fit.log
+susage.csv
+_OVERALL_USAGE.csv
 
 # Ignore all data subdirectory plots except those explicitly added
 data/*/filter.sh

.gitlab-ci.yml

@@ -28,11 +28,15 @@ plot_general:
     GIT_SUBMODULE_STRATEGY: recursive
   script:
     - mkdir -v plots
-    - cd plot_general
+    - cd plots_scripts
     - bash ./plot.sh
     - bash ./plot_replotgraphs.sh
+    - cd ../plots_kpi_scripts
+    - bash ./plot.sh
     - cd ..
     - pdfunite plots/pdf/*.pdf plots/all.pdf
+    - pdfunite plots_kpi/pdf/*.pdf plots_kpi/all.pdf
   artifacts:
     paths:
       - plots
+      - plots_kpi

README.md

@@ -6,7 +6,10 @@ The data is incorporated via submodules from separate projects.
 
 ##  Generated Plots (most recent)
 
-The plots are generated automatically when changes are detected or the generation pipeline is triggered.
+All plots are generated automatically when changes are detected or the generation pipeline is triggered.
+Also daily plots are generated.
+
+### General overview plots
 
 * [**Latest build, all graphs**](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/browse/plots?job=plot_general)
 * [Download all graphs in zip file](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/download/plots?job=plot_general)
@@ -14,20 +17,15 @@ The plots are generated automatically when changes are detected or the generatio
 * [Gallery view](gallery/gallery.md)
 * [Graphs pdf collection](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/all.pdf?job=plot_general)
 * Example of inline svg (AAI users):  
-  ![AAI users](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/aai_users-plot_2.svg?job=plot_general)
-
-
-## Services
+  ![AAI users](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/aai_users_per_centre-plot_cumulated_with_overall.svg?job=plot_general)
 
-This is the list of services currently being considered.
-
-* Helmholtz AAI
-* Helmholtz Backbone (VPN)
-* Helmholtz Codebase - (Gitlab at HZDR)
-* Nubes (HZB)
-* Mattermost (HZDR)
-* Openstack (FZJ)
-* bwSync&Share (KIT)
+### KPI plots
+* [**Latest build, all graphs**](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/browse/plots_kpi?job=plot_general)
+* [Download all graphs in zip file](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/download/plots_kpi?job=plot_general)
+* [Previously used graphs](example_graphs.md), used in reports, reproduced with most recent data.
+* [Graphs pdf collection](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots_kpi/all.pdf?job=plot_general)
+* Example of KPI plot (HZDR Codebase):  
+  ![HZDR codebase users](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots_kpi/svg/hzdr_codebase_active_users-plot_4.svg?job=plot_general)
 
 ## Local execution
 
@@ -45,7 +43,7 @@ git submodule update --remote --recursive
 Necessary for converting and plotting: `gnuplot`, `gawk`, `imagemagick`, `poppler-utils`.
 
 ```code
-cd plot_general      # or any other of the sub-folders
+cd plots_scripts      # or any other of the sub-folders
 ./plot.sh
 ```
 

data/desy_events/T_onboard.txt

+../../metadata/T_onboard/desy_events.txt
\ No newline at end of file

data/desy_notes_15min/T_onboard.txt

+../../metadata/T_onboard/desy_notes.txt
\ No newline at end of file

data/desy_notes_daily/T_onboard.txt

+../../metadata/T_onboard/desy_notes.txt
\ No newline at end of file

data/desy_notes_weekly/T_onboard.txt

+../../metadata/T_onboard/desy_notes.txt
\ No newline at end of file

data/desy_syncnshare_consumption/T_onboard.txt

+../../metadata/T_onboard/desy_syncnshare.txt
\ No newline at end of file

data/desy_syncnshare_users_aai/T_onboard.txt

+../../metadata/T_onboard/desy_syncnshare.txt
\ No newline at end of file

data/desy_syncnshare_users_all/T_onboard.txt

+../../metadata/T_onboard/desy_syncnshare.txt
\ No newline at end of file

data/desy_syncnshare_users_all/data.csv

+../../subprojects/DESY-syncnshare/stats/users_all.csv
\ No newline at end of file

data/kpi-weights.csv

+service_name,col_number,kpi,weight
+aai_active_users,2,Active User [number of active user],0
+aai_users,2,registered users,0
+desy_notes_15min,2,onlineNotes,0
+desy_notes_15min,3,onlineUsers,0
+desy_notes_15min,4,notesCount,0
+desy_notes_15min,5,registeredUsers,0
+desy_notes_15min,6,onlineRegisteredUsers,0
+desy_notes_daily,2,peak onlineNotes,0
+desy_notes_daily,3,peak onlineUsers,0
+desy_notes_daily,4,peak notesCount,0
+desy_notes_daily,5,peak registeredUsers,0
+desy_notes_daily,6,peak onlineRegisteredUsers,0
+desy_notes_weekly,2,peak onlineNotes,0
+desy_notes_weekly,3,peak onlineUsers,0
+desy_notes_weekly,4,peak notesCount,0
+desy_notes_weekly,5,peak registeredUsers,0
+desy_notes_weekly,6,peak onlineRegisteredUsers,0
+desy_syncnshare_users_aai,2,Users authenticated via Helmholtz AAI,0.5
+desy_syncnshare_users_all,2,All Users,0.5
+fzj_b2share,2,Disk Space [Byte],0
+fzj_b2share,3,Records [Number of records],0
+fzj_b2share,4,Files [Number of files],0
+fzj_jupyter,2,successful jobs,0.16667
+fzj_jupyter,3,unique users,0.5
+fzj_jupyter,4,nodes,0.16667
+fzj_jupyter,5,gpus,0.16667
+fzj_openstack,2,CPU Hours [h],0.2
+fzj_openstack,3,RAM MB-Hours [MB h],0.2
+fzj_openstack,4,Disk GB-Hours [GB h],0.2
+fzj_openstack,5,Servers,0.2
+fzj_openstack,6,Projects,0.2
+helmholtz-cloud-services,2,services [number],0
+hzb_nubes,2,active user,0.5
+hzb_nubes,3,folders,0
+hzb_nubes,4,files,0.125
+hzb_nubes,5,user-storage [G],0.125
+hzb_nubes,6,groupfolders – on top,0
+hzb_nubes,7,groupfolders,0.125
+hzb_nubes,8,Groupfolder-files,0
+hzb_nubes,9,group-storage [G],0
+hzb_nubes,10,shares,0.125
+hzdr_codebase_active_users,2,number_of_users,0.25
+hzdr_codebase_active_users,3,last_active_1_days,0
+hzdr_codebase_active_users,4,last_active_7_days,0.25
+hzdr_codebase_active_users,5,last_active_14_days,0
+hzdr_codebase_active_users,6,last_active_30_days,0
+hzdr_codebase_active_users,7,last_active_60_days,0
+hzdr_codebase_project_statistics,2,number_of_projects,0.125
+hzdr_codebase_project_statistics,3,total_commit_count,0.125
+hzdr_codebase_project_statistics,4,total_storage_size,0.125
+hzdr_codebase_project_statistics,5,total_repository_size,0
+hzdr_codebase_project_statistics,6,total_wiki_size,0
+hzdr_codebase_project_statistics,7,total_lfs_object_size,0
+hzdr_codebase_project_statistics,8,total_job_artifacts_size,0
+hzdr_codebase_project_statistics,9,last_active_1_days,0
+hzdr_codebase_project_statistics,10,last_active_7_days,0.125
+hzdr_codebase_project_statistics,11,last_active_14_days,0
+hzdr_codebase_project_statistics,12,last_active_30_days,0
+hzdr_codebase_project_statistics,13,last_active_60_days,0
+hzdr_helpdesk,2,number_of_created_tickets_per_7_days,1
+hzdr_mattermost,2, post count ,0.25
+hzdr_mattermost,3, team count ,0.25
+hzdr_mattermost,4, daily active users ,0
+hzdr_mattermost,5, monthly active users,0.5
+kit_bwsyncnshare,2,nc-serverinfo-api_nc-si.mean,0
+vpn_centres,2,vpn_centres [centres],0

gallery/gallery.md

@@ -89,8 +89,11 @@
 ##### desy_syncnshare_consumption-plot_4.svg
 ![desy_syncnshare_consumption-plot_4.svg](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/desy_syncnshare_consumption-plot_4.svg?job=plot_general)
 
-##### desy_syncnshare_users-plot_2.svg
-![desy_syncnshare_users-plot_2.svg](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/desy_syncnshare_users-plot_2.svg?job=plot_general)
+##### desy_syncnshare_users_aai-plot_2.svg
+![desy_syncnshare_users_aai-plot_2.svg](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/desy_syncnshare_users_aai-plot_2.svg?job=plot_general)
+
+##### desy_syncnshare_users_all-plot_2.svg
+![desy_syncnshare_users_all-plot_2.svg](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/desy_syncnshare_users_all-plot_2.svg?job=plot_general)
 
 ##### fzj_b2share-plot_2.svg
 ![fzj_b2share-plot_2.svg](https://gitlab.hzdr.de/hifis/overall/kpi/kpi-plots-ci/-/jobs/artifacts/master/raw/plots/svg/fzj_b2share-plot_2.svg?job=plot_general)

metadata/T_onboard/add_metadata.sh

+#!/bin/bash
+
+# for every text file in here, depicting one service onboarding time each....
+for i in *.txt; do
+
+    # ...we look for corresponding data files, with arbitary sub-KPI naming
+    for j in ../../data/"${i%.txt}"*; do
+
+        # if this is really a directory link data.
+        if [ -d "$j" ]; then
+            ln -sv ../../metadata/T_onboard/"$i" "$j"/T_onboard.txt
+        fi
+    done
+done
\ No newline at end of file

metadata/T_onboard/desy_events.txt

+T_onboard="2021-12-08" # according to https://gitlab.hzdr.de/hifis/cloud/access-layer/portal/-/tree/b27072f865a84f270cd23942c31059df4037ae98/cerebrum/data/marketService
\ No newline at end of file

metadata/T_onboard/desy_notes.txt

+T_onboard="2021-12-10" # according to https://gitlab.hzdr.de/hifis/cloud/access-layer/portal/-/tree/a40dc87391130c84aabeeed9dc3971c30193c63c/cerebrum/data/marketService
\ No newline at end of file

metadata/T_onboard/desy_syncnshare.txt

+T_onboard="2021-09-07" # according to https://gitlab.hzdr.de/hifis/cloud/access-layer/portal/-/blob/9e0283dad6987cc3dbabbb78d79e6cb0472b772b/cerebrum/data/marketService.json
\ No newline at end of file

plots_kpi_scripts/filter.sh

+../plots_scripts/filter.sh
\ No newline at end of file

plots_kpi_scripts/plot.plt

+#!/bin/gnuplot
+
+# Check if the separator is a "," or ";". We want to have at least two columns, so we'll test:
+sep=","
+set datafile separator sep
+stats 'data_plot.csv' nooutput
+# If we detected too few columns, try another separator.
+if (STATS_columns<=1) {
+  sep=";"
+  set datafile separator sep
+  stats 'data_plot.csv' nooutput
+}
+
+print "Detected ".STATS_columns." columns."
+
+if (STATS_columns<=1) {
+  print "Detected too few columns. Exiting."
+  exit
+}
+
+set yrange [0:*]
+set xdata time
+set format x "%Y-%m-%d"
+
+# There are chaotically many ways to encode date/time so far. We presume a standard format, but allow to override the format via timefmt.txt
+set timefmt "%Y-%m-%d %H:%M"
+if (system("[ ! -f timefmt.txt ]; echo $?")) {
+    load "timefmt.txt"
+}
+
+set xrange ["2020-01-01":"2022-04-02"]
+set xtics rotate by 45
+set xtics textcolor rgbcolor "black" offset -4,-3
+
+set  xtics '2020-01-01 00:00',365*24*3600
+set mxtics 12
+set grid xtics nomxtics ytics
+
+set key top left
+set linetype 2 dashtype 2
+
+
+# set some coordinates for diagram formatting. Can also be used to draw multiple aligned diagrams in one canvas, if wanted (multiplot)
+MP_LEFT = .1
+MP_RIGHT = .95
+MP_BOTTOM = .14
+MP_TOP = .93
+MP_xGAP = 0.1
+MP_yGAP = 0.02
+
+
+# fitting params
+set fit errorscaling errorvariables
+FIT_LIMIT=1e-15     # this should help to fit even very degenerate plots
+FIT_MAXITER=1000
+
+# fitting function
+susage(x) = m*x + y0
+
+# Plot all columns: One file for each column.
+
+filename='data_plot.csv'
+# since replot does not work reliably, we repeat the whole plot sequence
+# explicitly for every output format wanted.
+
+# log to one file per service
+set print "susage.csv"
+
+do for [i=2:STATS_columns] {
+
+    # starting parameters for each fit
+    y0=100
+    m=1
+
+    # fit slope for data points at least from 2020 (to exclude any possible data points from 2019 or before)
+    # for annual reporting 2021: AND fit only for data that is <=2021
+
+    fmt = "%Y-%m-%d"
+    set xdata time
+    set timefmt fmt
+
+    # reporting date (end of measurement period)
+    T_report="2021-12-31"   # hard-coded for every report
+
+    # starting dates.
+    T_logstart=system("cat ".filename." | tr -d '\r' | awk 'BEGIN{FS=\"".sep."\"}{if (NR>1 && NF>1) {print $1; exit;}}'")
+
+
+    # Onboarding date can be no earlier than the start of the pilot Helmholtz Cloud, i.e. 2021-03-29.
+    # https://hifis.net/news/2021/03/29/helmholtz-cloud-beta
+
+    T_onboard="2021-03-29"
+    # if there is a specific file determining another T_onboard for that service, defined manually, use that instead:
+    if (system("[ ! -f T_onboard.txt ]; echo $?")) {
+        load "T_onboard.txt"
+    }
+
+    if (strptime(fmt, T_onboard) > strptime(fmt, T_logstart)) {
+        T_fitstart_overall=T_onboard
+    } else {
+        T_fitstart_overall=T_logstart
+    }
+
+    # Do the fitting
+    fit [strptime(fmt, T_fitstart_overall):strptime(fmt, T_report)] susage(x) filename u 1:i via m,y0
+
+
+    # Later: Yearly procedure for yearly reports, not defined for 2021 annual report yet.
+    # Will take care of this later for reporting 2023 ff.
+    # But we need a placeholder here.
+    T_fitstart_yearly="N/A"
+    T_reference_yearly="N/A"
+
+    # Now we have a fit and can decide where to put the T_reference_overall.
+    # Per default, T_reference_overall should be equal to T_fitstart_overall.
+    # Exceptions MAY be that the KPI at this date is very low (close to zero), such that a division by this KPI makes no sense.
+    # In that case, a separate date MAY be defined by Cloud Cluster Management in T_reference_overall.txt.
+    T_reference_overall=T_fitstart_overall
+    if (system("[ ! -f T_reference_overall.txt ]; echo $?")) {
+        load "T_reference_overall.txt"
+    }
+
+    # Using T_reference_overall and T_report, the corresponding KPIs at these dates are calculated from the fit.
+    KPI_e=susage(strptime(fmt, T_report))
+    KPI_a=susage(strptime(fmt, T_reference_overall))
+
+    # Compute the raw dollar value
+    kpi_raw=KPI_e/KPI_a-1
+
+    # check if we actually want to consider this value (service was on board long enough, i.e. >=90 days)
+    if ((strptime(fmt, T_report)-strptime(fmt, T_fitstart_overall)) >= (90.*3600.*24.)) {
+        kpi_raw_title=sprintf("%.3g - subject to weighting.", kpi_raw);
+    } else {
+        kpi_raw_title=sprintf("%.3g - NOT CONSIDERED: less than 90 days data available.", kpi_raw);
+        kpi_raw=0;
+    }
+
+
+    # Extract KPI name. Seemingly, the header name of the column cannot be directly derived with gnuplot,
+    # instead we have to circle via shell and pipes (urgh)
+    kpi_name=system("head -n 1 ".filename." | tr -d '\r' | awk 'BEGIN{FS=\"".sep."\"}{print $".i."}'")
+
+    # Printing to table
+    logdata=sprintf("%s,%d,%s,%g,%g,%g,%g,,%s,%s,%s,%s,%s,%s,%s,%g,%g,%g\n",servicename,i,kpi_name,m,m_err,y0,y0_err,T_logstart,T_onboard,T_fitstart_overall,T_reference_overall,T_fitstart_yearly,T_reference_yearly,T_report,KPI_a,KPI_e,kpi_raw)
+    print logdata
+
+    # Plotting (once to pdf, once to svg)
+    do for [IDX = 0:1] {
+        if (IDX==0) {
+            set terminal pdf  color noenhanced size 7,5 dashed
+            set output 'plot_'.i.'.pdf'
+        } else {
+            set terminal svg noenhanced size 800,600
+            set output 'plot_'.i.'.svg'
+        }
+
+        set multiplot layout 1,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
+        set title servicename
+
+        # print linear fitting variables and errors
+        set label sprintf("m=%.2g +- %.3g (%.3g %%),\ny0=%.2g +- %.3g (%.3g %%)\nT_reference_overall=%s, T_report=%s\nkpi-raw=%s",m,m_err,m_err/m*100,y0,y0_err,y0_err/y0*100,T_reference_overall,T_report,kpi_raw_title) left at graph 0.1, graph 0.7
+
+        plot filename u 1:i w lp pt 7 ps 0.25 lc black title columnheader ,\
+             (x>=strptime(fmt, T_fitstart_overall) && x<= strptime(fmt, T_report)) ? susage(x) : 1/0 w l linewidth 4 lc "#55bb55" notitle ,\
+             '+' using (T_onboard):(susage(strptime(fmt, T_onboard)))                     w p pt 8  ps 1.5 lw 3 lc "#000000" title 'T_onboard' ,\
+             '+' using (T_logstart):(susage(strptime(fmt, T_logstart)))                   w p pt 2  ps 1.5 lw 3 lc "#888888" title 'T_logstart' ,\
+             '+' using (T_fitstart_overall):(susage(strptime(fmt, T_fitstart_overall)))   w p pt 6  ps 1.5 lw 3 lc "#00bb00" title 'T_fitstart_overall' ,\
+             '+' using (T_reference_overall):(susage(strptime(fmt, T_reference_overall))) w p pt 4  ps 1.5 lw 3 lc "#0000bb" title 'T_reference_overall' ,\
+             '+' using (T_report):(susage(strptime(fmt, T_report)))                       w p pt 10 ps 1.5 lw 3 lc "#ff0000" title 'T_report'
+
+        unset multiplot
+        unset label
+    }
+}