From c1fb4fcaa17db2912763efd030296bf6b01c4519 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Wed, 2 Nov 2022 14:29:49 +0000 Subject: [PATCH 001/114] Create trackqueries.q file with table schema --- code/handlers/trackqueries.q | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 code/handlers/trackqueries.q diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q new file mode 100644 index 000000000..c1bbe9cfa --- /dev/null +++ b/code/handlers/trackqueries.q @@ -0,0 +1,3 @@ +\d .queries + +queries:@[value;`queries;([]start:`timestamp$();end:`timestamp$();duration:`timespan$();user:`symbol$();ipa:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:())] From 9a4f356fb2cdf4bf0c65fadd9604ce96b5856879 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Wed, 2 Nov 2022 15:46:11 +0000 Subject: [PATCH 002/114] Rename column headings --- code/handlers/trackqueries.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index c1bbe9cfa..233bd32e9 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -1,3 +1,3 @@ \d .queries -queries:@[value;`queries;([]start:`timestamp$();end:`timestamp$();duration:`timespan$();user:`symbol$();ipa:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:())] +queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`timespan$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:())] From 442c4a2f1226109f1ad78ab83b7f08acff7e9d3a Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Mon, 7 Nov 2022 14:55:18 +0000 Subject: [PATCH 003/114] Add success column to query logging table --- code/handlers/trackqueries.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 233bd32e9..9cdfb5082 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -1,3 +1,3 @@ \d .queries -queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`timespan$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:())] +queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`timespan$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] From ac39c90167b4e32ca005803c3293c6943aa44aaa Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Wed, 9 Nov 2022 11:20:39 +0000 Subject: [PATCH 004/114] Add query logging functions and redefine .zs --- code/handlers/trackqueries.q | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 9cdfb5082..399d7d326 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -1,3 +1,13 @@ \d .queries +// Create a queries table to record query information queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`timespan$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] + +logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} + +logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} + +.z.pg:{cmd:x; .queries.logquery[.proc.cp[];@[value;x;.queries.logqueryerror[.proc.cp[];;cmd;startp]];cmd;startp:.proc.cp[]]} + +.z.ps:{cmd:x; .queries.logquery[.proc.cp[];@[value;x;.queries.logqueryerror[.proc.cp[];;cmd;startp]];cmd;startp:.proc.cp[]]} + From 8471fb5489289c52031bdec234c58e97b69651ca Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Thu, 10 Nov 2022 14:27:58 +0000 Subject: [PATCH 005/114] Change .z functions to fit with other handler scripts --- code/handlers/trackqueries.q | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 399d7d326..0a8b383a9 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -1,13 +1,12 @@ \d .queries // Create a queries table to record query information -queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`timespan$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] +queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] -logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} +logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} -logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} +logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} -.z.pg:{cmd:x; .queries.logquery[.proc.cp[];@[value;x;.queries.logqueryerror[.proc.cp[];;cmd;startp]];cmd;startp:.proc.cp[]]} - -.z.ps:{cmd:x; .queries.logquery[.proc.cp[];@[value;x;.queries.logqueryerror[.proc.cp[];;cmd;startp]];cmd;startp:.proc.cp[]]} +.z.pg:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]}.z.pg +.z.ps:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]}.z.ps From c746347be95877c7efd92a3e08ced9a5c9bce58b Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 11 Nov 2022 12:45:32 +0000 Subject: [PATCH 006/114] Added enable and ignore functions --- code/handlers/order.txt | 1 + code/handlers/trackqueries.q | 19 ++++++++++++++----- config/settings/default.q | 8 +++++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/code/handlers/order.txt b/code/handlers/order.txt index b7df23c61..f074c6e23 100644 --- a/code/handlers/order.txt +++ b/code/handlers/order.txt @@ -3,6 +3,7 @@ ldap.q writeaccess.q controlaccess.q permissions.q +trackqueries.q trackclients.q logusage.q apidetails.q diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 0a8b383a9..6e41e2fb1 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -1,12 +1,21 @@ \d .queries -// Create a queries table to record query information +//flags and variables +enabled:@[value;`enabled;1b] +ignore:@[value;`ignore;1b] +ignorelist:@[value;`ignorelist;(`upd;"upd")] + +//table schema queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] +//upsert successful query logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} +//upsert failed query +logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long*.001endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} -logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} - -.z.pg:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]}.z.pg +//customising .z.pg/ps +p1:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]} +//added for ignoring messages +p2:{if[ignore; if[0h=type y;if[any first[y]~/:ignorelist; :x@y]]]; p1[x;y]} -.z.ps:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]}.z.ps +if[enabled; .z.pg:(p1).z.pg; .z.ps:(p2).z.ps;]; diff --git a/config/settings/default.q b/config/settings/default.q index ec55d8591..d5127a3c5 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -41,6 +41,12 @@ MAXIDLE:`long$0D // handles which haven't been used in this length of time will AUTORECONNECT:0b // whether to reconnect to processes previously subscribed to checksubscriptionperiod:0D00:00:10 // how frequently to check subscriptions are still connected - 0D means don't check +//queries config +\d .queries +enabled:1b +ignore:1b +ignorelist:(`upd;"upd") + // Permissions configuration \d .pm enabled:0b @@ -183,4 +189,4 @@ SAVEFILE:`:KUTR.csv; // test results savefile // data striping \d .ds -numseg:0i; // default value for process.csv to overwrite and pubsub.q check \ No newline at end of file +numseg:0i; // default value for process.csv to overwrite and pubsub.q check From ed6c9b8c43d3d09997d0df7f6e22d935f9c4a743 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Fri, 11 Nov 2022 13:44:57 +0000 Subject: [PATCH 007/114] Change runtime to a long in schema --- code/handlers/trackqueries.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 6e41e2fb1..dc9b60b65 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -6,7 +6,7 @@ ignore:@[value;`ignore;1b] ignorelist:@[value;`ignorelist;(`upd;"upd")] //table schema -queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] +queries:@[value;`queries;([]starttime:`timestamp$();endtime:`long$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] //upsert successful query logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} From 1694b6878c3da80da8dd947d4618a43fcb374ba0 Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 11 Nov 2022 14:26:55 +0000 Subject: [PATCH 008/114] fixed bug in logqueryerror function --- code/handlers/trackqueries.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 6e41e2fb1..eeeee3cbc 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -11,7 +11,7 @@ queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtim //upsert successful query logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} //upsert failed query -logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long*.001endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} +logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} //customising .z.pg/ps p1:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]} From 18f36d7ada591beabc8c4730a13bcb07d78d3eb7 Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 11 Nov 2022 14:35:01 +0000 Subject: [PATCH 009/114] fixed bug in logqueryerror function --- code/handlers/trackqueries.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 6c575afeb..eeeee3cbc 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -6,7 +6,7 @@ ignore:@[value;`ignore;1b] ignorelist:@[value;`ignorelist;(`upd;"upd")] //table schema -queries:@[value;`queries;([]starttime:`timestamp$();endtime:`long$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] +queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] //upsert successful query logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} From cf6599b7d6dbb5ef3ad464810fea38277a81e523 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Mon, 14 Nov 2022 16:55:55 +0000 Subject: [PATCH 010/114] Add save down functionality --- code/handlers/trackqueries.q | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index eeeee3cbc..942809a1f 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -4,6 +4,8 @@ enabled:@[value;`enabled;1b] ignore:@[value;`ignore;1b] ignorelist:@[value;`ignorelist;(`upd;"upd")] +settimer:@[value;`settimer;0D00:00:10] +threshold:@[value;`threshold;50] //table schema queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] @@ -19,3 +21,22 @@ p1:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;star p2:{if[ignore; if[0h=type y;if[any first[y]~/:ignorelist; :x@y]]]; p1[x;y]} if[enabled; .z.pg:(p1).z.pg; .z.ps:(p2).z.ps;]; + +savedown:{[dir;tabname;pt] + pth:` sv .Q.par[dir;pt;tabname],`; + err:{[e].lg.e[`savedata;"Failed to save client query data to disk : ",e];'e}; + .[upsert;(pth;.Q.en[dir;.save.manipulate[tabname;.queries.queries]]);err]; + .queries.queries:0#.queries.queries; + }; + +rowcheck:{[dir;tabname;pt] + countvalue:count .queries.queries; + if[countvalue > .queries.threshold; savedown[dir;tabname;pt]]; + } + +settimers:{ + .timer.repeat[.proc.cp[];0Wp;.queries.settimer;(`.queries.rowcheck;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; + .timer.repeat[.proc.cp[];0Wp;0D00:01:00;(`.queries.savedown;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; + } + +if[enabled; .queries.settimers[];]; From 01d639a03a77a2be894a50b2c7afcdeeeed5c528 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 15 Nov 2022 11:29:31 +0000 Subject: [PATCH 011/114] Add timers to execute periodic save down --- code/handlers/trackqueries.q | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 942809a1f..cfd7c294d 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -4,7 +4,7 @@ enabled:@[value;`enabled;1b] ignore:@[value;`ignore;1b] ignorelist:@[value;`ignorelist;(`upd;"upd")] -settimer:@[value;`settimer;0D00:00:10] +timerval:@[value;`timerval;0D00:10:00] threshold:@[value;`threshold;50] //table schema @@ -22,21 +22,25 @@ p2:{if[ignore; if[0h=type y;if[any first[y]~/:ignorelist; :x@y]]]; p1[x;y]} if[enabled; .z.pg:(p1).z.pg; .z.ps:(p2).z.ps;]; +//functionality to save down tables to disk savedown:{[dir;tabname;pt] pth:` sv .Q.par[dir;pt;tabname],`; + numrows: count .queries.queries; + .lg.o[`save;"saving ",(string numrows)," rows of " (string tabname)," data to partition ", string pt] err:{[e].lg.e[`savedata;"Failed to save client query data to disk : ",e];'e}; - .[upsert;(pth;.Q.en[dir;.save.manipulate[tabname;.queries.queries]]);err]; + if[numrows > 0;.[upsert;(pth;.Q.en[dir;.save.manipulate[tabname;.queries.queries]]);err];]; .queries.queries:0#.queries.queries; }; rowcheck:{[dir;tabname;pt] countvalue:count .queries.queries; if[countvalue > .queries.threshold; savedown[dir;tabname;pt]]; - } - -settimers:{ - .timer.repeat[.proc.cp[];0Wp;.queries.settimer;(`.queries.rowcheck;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; - .timer.repeat[.proc.cp[];0Wp;0D00:01:00;(`.queries.savedown;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; - } + }; -if[enabled; .queries.settimers[];]; +//timers set up to periodically save down to disk +if[enabled; + settimers:{ + .timer.repeat[.proc.cp[];0Wp;0D00:00:10;(`.queries.rowcheck;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; + .timer.repeat[.proc.cp[];0Wp;.queries.timerval;(`.queries.savedown;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; + }; + .queries.settimers[];]; From 669aeb1be367760cf848f8896ca5571c1a49f864 Mon Sep 17 00:00:00 2001 From: acreehay Date: Tue, 15 Nov 2022 11:47:25 +0000 Subject: [PATCH 012/114] disabled query tracking --- config/settings/chainedtp.q | 3 +++ config/settings/segmentedchainedtickerplant.q | 3 +++ config/settings/segmentedtickerplant.q | 5 ++++- config/settings/tickerplant.q | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config/settings/chainedtp.q b/config/settings/chainedtp.q index ced208538..b4dc3585d 100644 --- a/config/settings/chainedtp.q +++ b/config/settings/chainedtp.q @@ -23,3 +23,6 @@ enabled:0b /- disable heartbeating /- Configuration used by the usage functions - logging of client interaction \d .usage enabled:0b /- switch off the usage logging + +\d .queries +enabled:0b /- disable query tracking diff --git a/config/settings/segmentedchainedtickerplant.q b/config/settings/segmentedchainedtickerplant.q index 051fe8b75..9e47c93fc 100644 --- a/config/settings/segmentedchainedtickerplant.q +++ b/config/settings/segmentedchainedtickerplant.q @@ -38,3 +38,6 @@ enabled:1b // switch on subscribercutoff \d .servers CONNECTIONS,:`segmentedtickerplant CONNECTIONSFROMDISCOVERY:1b + +\d .queries +enabled:0b // disable query tracking diff --git a/config/settings/segmentedtickerplant.q b/config/settings/segmentedtickerplant.q index 72c28bda6..ab3791e25 100644 --- a/config/settings/segmentedtickerplant.q +++ b/config/settings/segmentedtickerplant.q @@ -36,4 +36,7 @@ enabled:0b // disable the timer enabled:0b // disable heartbeating \d .zpsignore -enabled:0b // disable zpsignore - zps should be empty +enabled:0b // disable zpsignore - zps should be empty + +\d .queries +enabled:0b // disable query tracking diff --git a/config/settings/tickerplant.q b/config/settings/tickerplant.q index 5119ffcac..ef356a4ee 100644 --- a/config/settings/tickerplant.q +++ b/config/settings/tickerplant.q @@ -28,3 +28,5 @@ enabled:0b // disable heartbeating \d .zpsignore enabled:0b // disable zpsignore - zps should be empty +\d .queries +enabled:0b // disable query tracking From 19c4a286dc94abf07280a68ab8ec6aa463241ea9 Mon Sep 17 00:00:00 2001 From: acreehay Date: Tue, 15 Nov 2022 14:13:33 +0000 Subject: [PATCH 013/114] added ignore user function and ignore user list --- code/handlers/trackqueries.q | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index cfd7c294d..49d2d8e9f 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -1,28 +1,27 @@ \d .queries -//flags and variables +// flags and variables enabled:@[value;`enabled;1b] -ignore:@[value;`ignore;1b] -ignorelist:@[value;`ignorelist;(`upd;"upd")] -timerval:@[value;`timerval;0D00:10:00] +ignoreuser:@[value;`ignoreuser;1b] +ignoreuserlist:@[value;`ignoreuserlist;(`sort;`gateway;`monitor;`reporter;`segmentedtickerplant;`metrics;`dqc;`dqe;`;.z.u)] +timerval:@[value;`settimer;0D00:00:10] threshold:@[value;`threshold;50] -//table schema +// table schema queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] -//upsert successful query +// upsert successful query logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} -//upsert failed query +// upsert failed query logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} -//customising .z.pg/ps p1:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]} -//added for ignoring messages -p2:{if[ignore; if[0h=type y;if[any first[y]~/:ignorelist; :x@y]]]; p1[x;y]} +// for ignoring 'internal' messages +p2:{if[ignoreuser; if[any .z.u~/:ignoreuserlist; :x@y]]; p1[x;y]} -if[enabled; .z.pg:(p1).z.pg; .z.ps:(p2).z.ps;]; +if[enabled; .z.pg:(p2).z.pg; .z.ps:(p2).z.ps;]; -//functionality to save down tables to disk +// functionality to save down tables to disk savedown:{[dir;tabname;pt] pth:` sv .Q.par[dir;pt;tabname],`; numrows: count .queries.queries; @@ -37,7 +36,7 @@ rowcheck:{[dir;tabname;pt] if[countvalue > .queries.threshold; savedown[dir;tabname;pt]]; }; -//timers set up to periodically save down to disk +// timers set up to periodically save down to disk if[enabled; settimers:{ .timer.repeat[.proc.cp[];0Wp;0D00:00:10;(`.queries.rowcheck;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; From 6184e549f8f79b9c8f7ff7ce95d236ed9d6ab95e Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 18 Nov 2022 16:34:34 +0000 Subject: [PATCH 014/114] Added function for ignoring users and/or queries --- code/handlers/trackqueries.q | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 49d2d8e9f..2bdf15d41 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -2,8 +2,10 @@ // flags and variables enabled:@[value;`enabled;1b] +ignorequery:@[value;`ignorequery;1b] +ignorequerylist:@[value;`ignorequerylist;(`upd;)] ignoreuser:@[value;`ignoreuser;1b] -ignoreuserlist:@[value;`ignoreuserlist;(`sort;`gateway;`monitor;`reporter;`segmentedtickerplant;`metrics;`dqc;`dqe;`;.z.u)] +ignoreuserlist:@[value;`ignoreuserlist;(`discovery;`segmentedtickerplant;`rdb;`hdb;`sort;`gateway;`monitor;`housekeeping;`reporter;`filealerter;`feed;`segmentedchainedtickerplant;`sortworker;`metrics;`iexfeed;`dqc;`dqcdb;`dqe;`dqedb;`tailer_seg1;`tailer_seg2;`tr_seg1;`tr_seg2;`tailsort;`;.z.u)] timerval:@[value;`settimer;0D00:00:10] threshold:@[value;`threshold;50] @@ -16,8 +18,8 @@ logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$. logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} p1:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]} -// for ignoring 'internal' messages -p2:{if[ignoreuser; if[any .z.u~/:ignoreuserlist; :x@y]]; p1[x;y]} +// for ignoring users/procs and/or specific queries +p2:{if[ignoreuser; if[any .z.u~/:ignoreuserlist; :x@y]]; if[ignorequery; if[any y~/:ignorequerylist; :x@y]]; p1[x;y]} if[enabled; .z.pg:(p2).z.pg; .z.ps:(p2).z.ps;]; From a32e4db114f43cae5725cb90022566ab0007008d Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 18 Nov 2022 16:43:48 +0000 Subject: [PATCH 015/114] Replaced relative path to absolute in savedown function --- code/handlers/trackqueries.q | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q index 2bdf15d41..4ec848a03 100644 --- a/code/handlers/trackqueries.q +++ b/code/handlers/trackqueries.q @@ -24,7 +24,8 @@ p2:{if[ignoreuser; if[any .z.u~/:ignoreuserlist; :x@y]]; if[ignorequery; if[any if[enabled; .z.pg:(p2).z.pg; .z.ps:(p2).z.ps;]; // functionality to save down tables to disk -savedown:{[dir;tabname;pt] +savedown:{[tabname;pt] + dir:`$":",getenv `KDBHDB; pth:` sv .Q.par[dir;pt;tabname],`; numrows: count .queries.queries; .lg.o[`save;"saving ",(string numrows)," rows of " (string tabname)," data to partition ", string pt] @@ -33,15 +34,15 @@ savedown:{[dir;tabname;pt] .queries.queries:0#.queries.queries; }; -rowcheck:{[dir;tabname;pt] +rowcheck:{[tabname;pt] countvalue:count .queries.queries; - if[countvalue > .queries.threshold; savedown[dir;tabname;pt]]; + if[countvalue > .queries.threshold; savedown[tabname;pt]]; }; // timers set up to periodically save down to disk if[enabled; settimers:{ - .timer.repeat[.proc.cp[];0Wp;0D00:00:10;(`.queries.rowcheck;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; - .timer.repeat[.proc.cp[];0Wp;.queries.timerval;(`.queries.savedown;`:data/hdb;`clientqueries;.z.d);"save client query data to disk"]; + .timer.repeat[.proc.cp[];0Wp;0D00:00:10;(`.queries.rowcheck;`clientqueries;.z.d);"save client query data to disk"]; + .timer.repeat[.proc.cp[];0Wp;.queries.timerval;(`.queries.savedown;`clientqueries;.z.d);"save client query data to disk"]; }; .queries.settimers[];]; From 155cce8d2d9a421a791dfd586537de7bbb947ef6 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 29 Nov 2022 13:57:25 +0000 Subject: [PATCH 016/114] Add queryfeed process file --- code/processes/queryfeed.q | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 code/processes/queryfeed.q diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q new file mode 100644 index 000000000..8b1500d4c --- /dev/null +++ b/code/processes/queryfeed.q @@ -0,0 +1,25 @@ +us:@[value;`us;([]querytime:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$();procname:`symbol$;status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())]; + +upd:{[t;x] if [t in `.usage.usage; `us insert x]}; + +.servers.startup[]; +start_sub:{[subprocs] + hds:(),exec w from .servers.SERVERS where proctype in subprocs; + { + .lg.o[`startsub;"subscribing to ", string first exec procname from .servers.SERVERS where w=x]; + x(`.u.sub;`.usage.usage;`); + .lg.o[`completesub;"subscribed"]; + + }each hds; + }; +start_sub[subprocs]; + +queryfeed:{ + h(".u.upd";`usage;value flip select from us); + us::0#us; + }; + +.servers.startupdepcycles[`qtp;10;0W]; +h:.servers.gethandlebytype[`qtp;`any]; + +.timer.repeat[.proc.cp[];0Wp;0D00:00:00.200;(`queryfeed;`);"Publish Query Feed"]; From ef38603f0d621a1459e21ac2a0e2914da0f5a3b4 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 29 Nov 2022 14:01:53 +0000 Subject: [PATCH 017/114] Add query process config files --- config/settings/qtp.q | 43 +++++++++++++++++++++++++++++++++++++ config/settings/queryfeed.q | 7 ++++++ config/settings/queryrdb1.q | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 config/settings/qtp.q create mode 100644 config/settings/queryfeed.q create mode 100644 config/settings/queryrdb1.q diff --git a/config/settings/qtp.q b/config/settings/qtp.q new file mode 100644 index 000000000..b4477eafc --- /dev/null +++ b/config/settings/qtp.q @@ -0,0 +1,43 @@ +// Segmented TP config + +\d .stplg + +multilog:`tabperiod; // [tabperiod|none|periodic|tabular|custom] +multilogperiod:0D01; // Length of period for STP periodic logging modes +errmode:1b; // Enable error mode for STP +batchmode:`defaultbatch; // [memorybatch|defaultbatch|immediate] +replayperiod:`day // [period|day|prior] +customcsv:hsym first .proc.getconfigfile["stpcustom.csv"]; // Location for custom logging mode csv + +\d .proc +loadcommoncode:0b // do not load common code +loadprocesscode:1b // load process code +logroll:0b // do not roll logs + +// Configuration used by the usage functions - logging of client interaction +\d .usage +enabled:0b // switch off the usage logging + +// Client tracking configuration +// This is the only thing we want to do +// and only for connections being opened and closed +\d .clients +enabled:1b // whether client tracking is enabled +opencloseonly:1b // only log open and closing of connections + +// Server connection details +\d .servers +enabled:0b // disable server tracking + +\d .timer +enabled:0b // disable the timer + +\d .hb +enabled:0b // disable heartbeating + +\d .zpsignore +enabled:0b // disable zpsignore - zps should be empty + +\d .queries +enabled:0b + diff --git a/config/settings/queryfeed.q b/config/settings/queryfeed.q new file mode 100644 index 000000000..1b7024c4f --- /dev/null +++ b/config/settings/queryfeed.q @@ -0,0 +1,7 @@ +\d .servers +enabled:1b +CONNECTIONS:`qtp`rdb`hdb // Feedhandler connects to the tickerplant +HOPENTIMEOUT:30000 + +\d . +subprocs:`rdb`hdb diff --git a/config/settings/queryrdb1.q b/config/settings/queryrdb1.q new file mode 100644 index 000000000..32bc00312 --- /dev/null +++ b/config/settings/queryrdb1.q @@ -0,0 +1,40 @@ +// Bespoke RDB config + +\d .rdb +ignorelist:`heartbeat`logmsg //list of tables to ignore when saving to disk +hdbtypes:`hdb //list of hdb types to look for and call in hdb reload +hdbnames:() //list of hdb names to search for and call in hdb reload +tickerplanttypes:`qtp //list of tickerplant types to try and make a connection to +gatewaytypes:`gateway //list of gateway types to try and make a connection to +checktpperiod:0D00:00:05 //how often to check for tickerplant connection +onlyclearsaved:0b //if true, eod writedown will only clear tables which have been successfully saved to disk +subscribeto:` //a list of tables to subscribe to, default (`) means all tables +subscribesyms:` //a list of syms to subscribe for, (`) means all syms +savetables:1b //if true tables will be saved at end of day, if false tables wil not be saved, only wiped +garbagecollect:1b //if true .Q.gc will be called after each writedown - tradeoff: latency vs memory usage +upd:insert //value of upd +hdbdir:`:hdb //the location of the hdb directory +replaylog:1b //replay the tickerplant log file +schema:1b //retrieve the schema from the tickerplant +tpconnsleepintv:10 //number of seconds between attempts to connect to the tp +gc:1b //if true .Q.gc will be called after each writedown - tradeoff: latency vs memory usage + +sortcsv:hsym first .proc.getconfigfile["sort.csv"] //location of csv file +reloadenabled:0b //if true, the RDB will not save when .u.end is called but + //will clear it's data using reload function (called by the WDB) +parvaluesrc:`log //where to source the rdb partition value, can be log (from tp log file name), + //tab (from the the first value in the time column of the table that is subscribed for) + //anything else will return a null date which is will be filled by pardefault +pardefault:.z.D //if the src defined in parvaluesrc returns null, use this default date instead +tpcheckcycles:0W //specify the number of times the process will check for an available tickerplant +subfiltered:0b //allows subscription filters to be loaded and applied in the rdb +connectonstart:1b //rdb connects to tickerplant as soon as it is started + +\d .proc +loadprocesscode:1b // whether to load the process specific code defined at ${KDBCODE}/{process type} + +// Server connection details +\d .servers +CONNECTIONS:`hdb // list of connections to make at start up +STARTUP:1b // create connections + From efb9e7e306e609fdf6d79b43db27309df12b7322 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 29 Nov 2022 14:04:27 +0000 Subject: [PATCH 018/114] Add qtp code directory --- code/qtp/pubsub.q | 30 +++++ code/qtp/sctp.q | 57 +++++++++ code/qtp/stplog.q | 296 +++++++++++++++++++++++++++++++++++++++++++++ code/qtp/stpmeta.q | 49 ++++++++ 4 files changed, 432 insertions(+) create mode 100644 code/qtp/pubsub.q create mode 100644 code/qtp/sctp.q create mode 100644 code/qtp/stplog.q create mode 100644 code/qtp/stpmeta.q diff --git a/code/qtp/pubsub.q b/code/qtp/pubsub.q new file mode 100644 index 000000000..2f05ff55c --- /dev/null +++ b/code/qtp/pubsub.q @@ -0,0 +1,30 @@ +// Get pubsub common code +.proc.loadf[getenv[`KDBCODE],"/common/pubsub.q"]; + +// Define UPD and ZTS wrapper functions + +// Check for end of day/period and call inner UPD function +.stpps.upd.def:{[t;x] + if[.stplg.nextendUTC`create); // dict r contains icounts & not using own logfile + subtabs:$[subscribeto~`;key r`icounts;subscribeto],(); + .u.jcounts::.u.icounts::$[0=count r`icounts;()!();subtabs!enlist [r`icounts]subtabs]; + ] + ]; + } + +// Initialise chained STP +init:{ + // Load in timer and subscription code and set top-level end of day/period functions + .proc.loadf[getenv[`KDBCODE],"/common/timer.q"]; + .proc.loadf[getenv[`KDBCODE],"/common/subscriptions.q"]; + `endofperiod set {[x;y;z] .stplg.endofperiod[x;y;z]}; + `endofday set {[x;y] .stplg.endofday[x;y]}; + + // Initialise connections and subscribe to main STP + .servers.startupdepnamecycles[.sctp.tickerplantname;.sctp.tpconnsleep;.sctp.tpcheckcycles]; + .sctp.subscribe[]; + }; + +\d . + +// Make the SCTP die if the main STP dies +.z.pc:{[f;x] + @[f;x;()]; + if[.sctp.chainedtp; + if[.sctp.tph=x; .lg.e[`.z.pc;"lost connection to tickerplant : ",string .sctp.tickerplantname];exit 1] + ] + } @[value;`.z.pc;{{}}]; + +// Extract data from incoming table as a list +upd:{[t;x] + x:value flip x; + .u.upd[t;x] + } \ No newline at end of file diff --git a/code/qtp/stplog.q b/code/qtp/stplog.q new file mode 100644 index 000000000..ff08f6e89 --- /dev/null +++ b/code/qtp/stplog.q @@ -0,0 +1,296 @@ +// Utilites for periodic tp logging in stp process + +// Live logs and handles to logs for each table +currlog:([tbl:`symbol$()]logname:`symbol$();handle:`int$()) + +// View of log file handles for faster lookups +loghandles::exec tbl!handle from currlog + +\d .stplg + +// Name of error log file +errorlogname:@[value;`.stplg.errorlogname;`segmentederrorlogfile] + +// Create stp log directory +// Log structure `:stplogs/date/tabname_time +createdld:{[name;date] + if[not count dir:hsym `$getenv[`KDBTPLOG];.lg.e[`stp;"log directory not defined"];exit 1]; + .os.md dir; + .os.md .stplg.dldir:` sv dir,`$raze/[string name,"_",date]; + }; + +// Functions to generate log names in one of five modes + +// Generate standardised timestamp string for log names +gentimeformat:{(raze string "dv"$x) except ".:"}; + +// Tabperiod mode - TP log rolled periodically (default 1 hr), 1 log per table (default setting) +.stplg.logname.tabperiod:{[dir;tab;p] ` sv (hsym dir;`$raze string (.proc.procname;"_";tab),.stplg.gentimeformat[p]) }; + +// Standard TP mode - write all tables to single log, roll daily +.stplg.logname.singular:{[dir;tab;p] ` sv (hsym dir;`$raze string .proc.procname,"_",.stplg.gentimeformat[p]) }; + +// Periodic-only mode - write all tables to single log, roll periodically intraday +.stplg.logname.periodic:{[dir;tab;p] ` sv (hsym dir;`$raze string .proc.procname,"_periodic",.stplg.gentimeformat[p]) }; + +// Tabular-only mode - write tables to separate logs, roll daily +.stplg.logname.tabular:{[dir;tab;p] ` sv (hsym dir;`$raze string (.proc.procname;"_";tab),.stplg.gentimeformat[p]) }; + +// Custom mode - mixed periodic/tabular mode +// Tables are defined as periodic, tabular, tabperiod or none in config file stpcustom.csv +// Tables not specified in csv are not logged +.stplg.logname.custom:{[dir;tab;p] .stplg.logname[.stplg.custommode tab][dir;tab;p] }; + +// If in error mode, create an error log name using .stplg.errorlogname +.stplg.logname.error:{[dir;ename;p] ` sv (hsym dir;`$raze string (.proc.procname;"_";ename),.stplg.gentimeformat[p]) }; + +// Update and timer functions in three batch modes //////////////////////////////////// +// preserve pre-existing definitions +upd:@[value;`.stplg.upd;enlist[`]!enlist ()]; +zts:@[value;`.stplg.zts;enlist[`]!enlist ()]; + +// Functions to add columns on updates +updtab:@[value;`.stplg.updtab;enlist[`]!enlist {(enlist(count first x)#y),x}] + +// If set to memorybatch, publish and write to disk will be run in batches +// insert to table in memory, on a timer flush the table to disk and publish, update counts +upd[`memorybatch]:{[t;x;now] + t insert updtab[t] . (x;now); + }; + +zts[`memorybatch]:{ + {[t] + if[count value t; + `..loghandles[t] enlist (`upd;t;value flip value t); + @[`.stplg.msgcount;t;+;1]; + @[`.stplg.rowcount;t;+;count value t]; + .stpps.pubclear[t]]; + }each .stpps.t; + }; + +// Standard batch mode - write to disk immediately, publish in batches +upd[`defaultbatch]:{[t;x;now] + t insert x:.stplg.updtab[t] . (x;now); + `..loghandles[t] enlist(`upd;t;x); + // track tmp counts, and add these after publish + @[`.stplg.tmpmsgcount;t;+;1]; + @[`.stplg.tmprowcount;t;+;count first x]; + }; + +zts[`defaultbatch]:{ + // publish and clear all tables, increment counts + .stpps.pubclear[.stpps.t]; + // after data has been published, updated the counts + .stplg.msgcount+:.stplg.tmpmsgcount; + .stplg.rowcount+:.stplg.tmprowcount; + // reset temp counts + .stplg.tmpmsgcount:.stplg.tmprowcount:()!(); + }; + +// Immediate mode - publish and write immediately +upd[`immediate]:{[t;x;now] + x:updtab[t] . (x;now); + `..loghandles[t] enlist(`upd;t;x); + x:$[0h>type last x;enlist;flip] .stpps.tabcols[t]!x; + @[`.stplg.msgcount;t;+;1]; + @[`.stplg.rowcount;t;+;count x]; + .stpps.pub[t;x] + }; + +zts[`immediate]:{} + +////////////////////////////////////////////////////////////////////////////////////// + +// Functions to obtain logs for client replay //////////////////////////////////////// +// replaylog called from client-side, returns nested list of logcounts and lognames +replaylog:{[t] + getlogs[replayperiod][t] + } + +// alternative replay allows for 'pass through logging' +// if SCTP not producing logs, subscribers replay from STP log files +if[.sctp.loggingmode=`parent; + replaylog:{[t] + .sctp.tph (`.stplg.replaylog; t) + } + ] + +getlogs:enlist[`]!enlist () + +// If replayperiod set to `period, only replay logs for current logging period +getlogs[`period]:{[t] + distinct flip (.stplg.msgcount;exec tbl!logname from `..currlog where tbl in t)@\:t + }; + +// If replayperiod set to `day, replay all of today's logs +getlogs[`day]:{[t] + // set the msgcount to 0Wj for all logs which have closed + lnames:select seq,tbls,logname,msgcount:0Wj from .stpm.metatable where any each tbls in\: t; + // Meta table does not store counts for live logs, so these are populated here + lnames:update msgcount:sum each .stplg.msgcount[tbls] from lnames where seq=.stplg.i; + flip value exec `long$msgcount,logname from lnames + }; + +////////////////////////////////////////////////////////////////////////////////////// + +// Open log for a single table at start of logging period +openlog:{[multilog;dir;tab;p] + lname:logname[multilog][dir;tab;p]; + .lg.o[`openlog;"opening logfile: ",string lname]; + h:$[(notexists:not type key lname)or null h0:exec first handle from `..currlog where logname=lname; + [if[notexists;.[lname;();:;()]];hopen lname]; + h0 + ]; + `..currlog upsert (tab;lname;h); + }; + +// Error log for failed updates in error mode +openlogerr:{[dir] + lname:.[.stplg.logname.error;(dir;.stplg.errorlogname;.z.p+.eodtime.dailyadj);{.lg.e[`openlogerr;"failed to make error log: ",x]}]; + if[not type key lname;.[lname;();:;()]]; + h:@[{hopen x};lname;{.lg.e[`openlogerr;"failed to open handle to error log with error: ",x]}]; + `..currlog upsert (errorlogname;lname;h); + }; + +// Log failed message and error type in error mode +badmsg:{[e;t;x] + .lg.o[`upd;"Bad message received, error: ",e]; + `..loghandles[errorlogname] enlist(`upderr;t;x); + }; + +closelog:{[tab] + if[null h:`..currlog[tab;`handle];.lg.o[`closelog;"no open handle to log file"];:()]; + .lg.o[`closelog;"closing log file ",string `..currlog[tab;`logname]]; + @[hclose;h;{.lg.e[`closelog;"handle already closed"]}]; + update handle:0N from `..currlog where tbl=tab; + }; + +// Roll all logs at end of logging period +rolllog:{[multilog;dir;tabs;p] + .stpm.updmeta[multilog][`close;tabs;p]; + closelog each tabs; + @[`.stplg.msgcount;tabs;:;0]; + {[m;d;t] + .[openlog;(m;d;t;currperiod); + {.lg.e[`stp;"failed to open log for table ",string[y],": ",x]}[;t]] + }[multilog;dir;]each tabs; + .stpm.updmeta[multilog][`open;tabs;p]; + }; + +// Creates dictionary of process data to be used at endofday/endofperiod - configurable but default provided +endofdaydata:@[value;`.stplg.endofdaydata;{ {`proctype`procname`tables!(.proc.proctype;.proc.procname;.stpps.t)} }]; + +// endofperiod function defined in SCTP +// passes on eop messages to subscribers and rolls logs +endofperiod:{[currentpd;nextpd;data] + .lg.o[`endofperiod;"flushing remaining data to subscribers and clearing tables"]; + .stpps.pubclear[.stplg.t]; + .lg.o[`endofperiod;"executing end of period for ",.Q.s1 `currentperiod`nextperiod!(currentpd;nextpd)]; + .stpps.endp[currentpd;nextpd;data]; // sends endofperiod message to subscribers + currperiod::nextpd; // increments current period + if[.sctp.loggingmode=`create;periodrollover[data]] // logs only rolled if in create mode + }; + +// stp runs function to send out end of period messages and roll logs +// eop log roll is stopped if eod is also going to be triggered (roll is not stopped in SCTP) +stpeoperiod:{[currentpd;nextpd;data;rolllogs] + .lg.o[`stpeoperiod;"passing on endofperiod message to subscribers"]; + .stpps.endp[currentpd;nextpd;data]; // sends endofperiod message to subscribers + currperiod::nextperiod; // increments current period + if[(data`p)>nextperiod::multilogperiod+currperiod; + system"t 0";'"next period is in the past"]; // timer off + getnextendUTC[]; // grabs next end time + if[rolllogs;periodrollover[data]]; // roll if appropriate + .lg.o[`stpeoperiod;"end of period complete, new values for current and next period are ",.Q.s1 (currentpd;nextpd)]; + } + +// common eop log rolling logic for STP and SCTP +periodrollover:{[data] + i+::1; // increments log seq number + rolllog[multilog;dldir;rolltabs;data`p]; + } + +// endofday function defined in SCTP +// passes on eod messages to subscribers and rolls logs +endofday:{[date;data] + .lg.o[`endofday;"flushing remaining data to subscribers and clearing tables"]; + .stpps.pubclear[.stplg.t]; + .stpps.end[date;data]; // sends endofday message to subscribers + dayrollover[data]; + } + +// STP runs function to send out eod messages and roll logs +stpeod:{[date;data] + .lg.o[`stpeod;"executing end of day for ",.Q.s1 .eodtime.d]; + .stpps.end[date;data]; // sends endofday message to subscribers + dayrollover[data]; + } + +// common eod log rolling logic for STP and SCTP +dayrollover:{[data] + if[(data`p)>.eodtime.nextroll:.eodtime.getroll[data`p]; + system"t 0";'"next roll is in the past"]; // timer off + getnextendUTC[]; // grabs next end time + .eodtime.d+:1; // increment current day + .stpm.updmeta[multilog][`close;logtabs;(data`p)+.eodtime.dailyadj]; // update meta tables + .stpm.metatable:0#.stpm.metatable; + closelog each logtabs; // close current day logs + init[string .proc.procname]; // reinitialise process + .lg.o[`dayrollover;"end of day complete, new value for date is ",.Q.s1 .eodtime.d]; + } + +// get the next end time to compare to +getnextendUTC:{nextendUTC::-1+min(.eodtime.nextroll;nextperiod - .eodtime.dailyadj)} + +checkends:{ + // jump out early if don't have to do either + if[nextendUTC > x; :()]; + // check for endofperiod + if[nextperiod < x1:x+.eodtime.dailyadj; stpeoperiod[.stplg`currperiod;.stplg`nextperiod;.stplg.endofdaydata[],(enlist `p)!enlist x1;not .eodtime.nextroll < x]]; + // check for endofday + if[.eodtime.nextroll < x;if[.eodtime.d<("d"$x)-1;system"t 0";'"more than one day?"]; stpeod[.eodtime.d;.stplg.endofdaydata[],(enlist `p)!enlist x]]; + }; + +init:{[dbname] + t::tables[`.]except `currlog; + msgcount::rowcount::t!count[t]#0; + tmpmsgcount::tmprowcount::(`symbol$())!`long$(); + logtabs::$[multilog~`custom;key custommode;t]; + rolltabs::$[multilog~`custom;logtabs except where custommode in `tabular`singular;t]; + currperiod::multilogperiod xbar .z.p+.eodtime.dailyadj; + nextperiod::multilogperiod+currperiod; + getnextendUTC[]; + i::1; + seqnum::0; + + if[(value `..createlogs) or .sctp.loggingmode=`create; + createdld[dbname;.eodtime.d]; + openlog[multilog;dldir;;.z.p+.eodtime.dailyadj]each logtabs; + // If appropriate, roll error log + if[.stplg.errmode;openlogerr[dldir]]; + // read in the meta table from disk + .stpm.metatable:@[get;hsym`$string[.stplg.dldir],"/stpmeta";0#.stpm.metatable]; + // set log sequence number to the max of what we've found + i::1+ -1|exec max seq from .stpm.metatable; + // add the info to the meta table + .stpm.updmeta[multilog][`open;logtabs;.z.p+.eodtime.dailyadj]; + ] + + // set loghandles to null if sctp is not creating logs + if[.sctp.chainedtp and not .sctp.loggingmode=`create; + `..loghandles set t! (count t) # enlist (::) + ] + }; + +\d . + +// Close logs on clean exit +.z.exit:{ + if[not x~0i;.lg.e[`stpexit;"Bad exit!"];:()]; + .lg.o[`stpexit;"Exiting process"]; + // exit before logs are touched if process is an sctp NOT in create mode + if[.sctp.chainedtp and not .sctp.loggingmode=`create; :()]; + .lg.o[`stpexit;"Closing off log files"]; + .stpm.updmeta[.stplg.multilog][`close;.stpps.t;.z.p]; + .stplg.closelog each .stpps.t; + } diff --git a/code/qtp/stpmeta.q b/code/qtp/stpmeta.q new file mode 100644 index 000000000..7adf3f4ed --- /dev/null +++ b/code/qtp/stpmeta.q @@ -0,0 +1,49 @@ +// API for writing logfile meta data +// Metatable keeps info on all opened logs, the tables which feed each log, and the number of messages written to each log + +\d .stpm + +metatable:([]seq:`int$();logname:`$();start:`timestamp$();end:`timestamp$();tbls:();msgcount:`int$();schema:();additional:()) + +// Functions to update meta data for all logs in each logging mode +// Meta is updated only when opening and closing logs +updmeta:enlist[`]!enlist () + +updmeta[`tabperiod]:{[x;t;p] + getmeta[x;p;;]'[enlist each t;`..currlog[([]tbl:t)]`logname]; + setmeta[.stplg.dldir;metatable]; + }; + +updmeta[`singular]:{[x;t;p] + getmeta[x;p;t;`..currlog[first t]`logname]; + setmeta[.stplg.dldir;metatable]; + }; + +updmeta[`periodic]:updmeta[`singular] + +updmeta[`tabular]:updmeta[`tabperiod] + +updmeta[`custom]:{[x;t;p] + pertabs:where `periodic=.stplg.custommode; + updmeta[`periodic][x;t inter pertabs;p]; + updmeta[`tabular][x;t except pertabs;p] + }; + +// Logname, start time, table names and schema populated on opening +// End time and final message count updated on close +// Sequence number increments by one on log period rollover +getmeta:{[x;p;t;ln] + if[x~`open; + s:((),t)!(),.stpps.schemas[t]; + `.stpm.metatable upsert (.stplg.i;ln;p;0Np;t;0;s;enlist ()!()); + ]; + if[x~`close; + update end:p,msgcount:sum .stplg.msgcount[t] from `.stpm.metatable where logname = ln + ] + }; + +setmeta:{[dir;mt] + t:(hsym`$string[dir],"/stpmeta"); + .[{x set y};(t;mt);{.lg.e[`setmeta;"Failed to set metatable with error: ",x]}]; + }; + From 162608d8447e199d3d26bac58e586cd6ae216d43 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Thu, 1 Dec 2022 12:31:22 +0000 Subject: [PATCH 019/114] Add usage log reload --- code/processes/queryfeed.q | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index 8b1500d4c..37dc037a6 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -14,12 +14,27 @@ start_sub:{[subprocs] }; start_sub[subprocs]; +readlog:{[file] + + // Remove leading backtick from symbol columns, convert a and w columns back to integers + update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, + a:"I"$-1 _' a, w:"I"$-1 _' w from + // Read in file + @[{update "J"$'" " vs' mem from flip (cols `.queries.us)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]}; + queryfeed:{ h(".u.upd";`usage;value flip select from us); us::0#us; }; +flushreload:{ + .lg.o[`flushreload1;"fr1"]; + procnames:exec distinct procname from .servers.SERVERS where proctype in subprocs; + {h(".u.upd";`usage;value flip select from readlog[raze string (getenv `KDBLOG),"/usage_",(raze x),"_",.z.d,".log"])} each string each procnames; + }; + .servers.startupdepcycles[`qtp;10;0W]; h:.servers.gethandlebytype[`qtp;`any]; +.timer.once[.proc.cp[]+0D00:00:10.000;(`flushreload;`);"Flush reload"] .timer.repeat[.proc.cp[];0Wp;0D00:00:00.200;(`queryfeed;`);"Publish Query Feed"]; From 58f220c3dd8353a2ef651a83729bc14a868d3755 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Thu, 1 Dec 2022 16:10:57 +0000 Subject: [PATCH 020/114] Remove typo --- code/processes/queryfeed.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index 37dc037a6..7dbbe3632 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -20,7 +20,7 @@ readlog:{[file] update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, a:"I"$-1 _' a, w:"I"$-1 _' w from // Read in file - @[{update "J"$'" " vs' mem from flip (cols `.queries.us)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]}; + @[{update "J"$'" " vs' mem from flip (cols `us)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]}; queryfeed:{ h(".u.upd";`usage;value flip select from us); From f7dab3575d5daff97558690580224c05d6f05645 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 6 Dec 2022 15:18:46 +0000 Subject: [PATCH 021/114] Add enabling functionality for log reload --- code/processes/queryfeed.q | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index 7dbbe3632..14223d109 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -1,17 +1,17 @@ us:@[value;`us;([]querytime:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$();procname:`symbol$;status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())]; -upd:{[t;x] if [t in `.usage.usage; `us insert x]}; +upd:{[t;x] if [t in `usage; `us insert x]}; .servers.startup[]; start_sub:{[subprocs] hds:(),exec w from .servers.SERVERS where proctype in subprocs; { .lg.o[`startsub;"subscribing to ", string first exec procname from .servers.SERVERS where w=x]; - x(`.u.sub;`.usage.usage;`); + x(`.u.sub;`usage;`); .lg.o[`completesub;"subscribed"]; - }each hds; }; + start_sub[subprocs]; readlog:{[file] @@ -23,7 +23,7 @@ readlog:{[file] @[{update "J"$'" " vs' mem from flip (cols `us)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]}; queryfeed:{ - h(".u.upd";`usage;value flip select from us); + h(".u.upd";`usage;value flip select from `us); us::0#us; }; @@ -36,5 +36,5 @@ flushreload:{ .servers.startupdepcycles[`qtp;10;0W]; h:.servers.gethandlebytype[`qtp;`any]; -.timer.once[.proc.cp[]+0D00:00:10.000;(`flushreload;`);"Flush reload"] +if[reloadenabled;.timer.once[.proc.cp[]+0D00:00:10.000;(`flushreload;`);"Flush reload"]]; .timer.repeat[.proc.cp[];0Wp;0D00:00:00.200;(`queryfeed;`);"Publish Query Feed"]; From 14fa853cce018e1d6d0165dee1b98cb0f86fe85b Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 6 Dec 2022 15:24:07 +0000 Subject: [PATCH 022/114] Add log reload config variable --- config/settings/queryfeed.q | 1 + 1 file changed, 1 insertion(+) diff --git a/config/settings/queryfeed.q b/config/settings/queryfeed.q index 1b7024c4f..d6cb85e4a 100644 --- a/config/settings/queryfeed.q +++ b/config/settings/queryfeed.q @@ -5,3 +5,4 @@ HOPENTIMEOUT:30000 \d . subprocs:`rdb`hdb +reloadenabled:1b From e054bc58ce2960f0193d4fd8d522a89c041695da Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 6 Dec 2022 15:45:11 +0000 Subject: [PATCH 023/114] Add queryrdb config file --- config/settings/queryrdb.q | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 config/settings/queryrdb.q diff --git a/config/settings/queryrdb.q b/config/settings/queryrdb.q new file mode 100644 index 000000000..9ad9e3bd0 --- /dev/null +++ b/config/settings/queryrdb.q @@ -0,0 +1,44 @@ +// Bespoke RDB config + +\d .rdb +ignorelist:`heartbeat`logmsg`.usage.usage //list of tables to ignore when saving to disk +hdbtypes:`queryhdb //list of hdb types to look for and call in hdb reload +hdbnames:() //list of hdb names to search for and call in hdb reload +tickerplanttypes:`qtp //list of tickerplant types to try and make a connection to +gatewaytypes:`gateway //list of gateway types to try and make a connection to +checktpperiod:0D00:00:05 //how often to check for tickerplant connection +onlyclearsaved:0b //if true, eod writedown will only clear tables which have been successfully saved to disk +subscribeto:` //a list of tables to subscribe to, default (`) means all tables +subscribesyms:` //a list of syms to subscribe for, (`) means all syms +savetables:1b //if true tables will be saved at end of day, if false tables wil not be saved, only wiped +garbagecollect:1b //if true .Q.gc will be called after each writedown - tradeoff: latency vs memory usage +upd:insert //value of upd +hdbdir:hsym `$getenv[`KDBQUERYHDB] //the location of the hdb directory +replaylog:1b //replay the tickerplant log file +schema:1b //retrieve the schema from the tickerplant +tpconnsleepintv:10 //number of seconds between attempts to connect to the tp +gc:1b //if true .Q.gc will be called after each writedown - tradeoff: latency vs memory usage + +sortcsv:hsym first .proc.getconfigfile["querysort.csv"] //location of csv file +reloadenabled:0b //if true, the RDB will not save when .u.end is called but + //will clear it's data using reload function (called by the WDB) +parvaluesrc:`log //where to source the rdb partition value, can be log (from tp log file name), + //tab (from the the first value in the time column of the table that is subscribed for) + //anything else will return a null date which is will be filled by pardefault +pardefault:.z.D //if the src defined in parvaluesrc returns null, use this default date instead +tpcheckcycles:0W //specify the number of times the process will check for an available tickerplant +subfiltered:0b //allows subscription filters to be loaded and applied in the rdb +connectonstart:1b //rdb connects to tickerplant as soon as it is started + +\d .proc +loadprocesscode:1b // whether to load the process specific code defined at ${KDBCODE}/{process type} + +// Server connection details +\d .servers +CONNECTIONS:`queryhdb // list of connections to make at start up +STARTUP:1b // create connections + +\d .usage +enabled:0b +querytrack:0b + From ccec51bce2087d45071bf5fb868e3e1cbee1712d Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 6 Dec 2022 15:55:25 +0000 Subject: [PATCH 024/114] Add queryhdb config file --- config/settings/queryhdb.q | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 config/settings/queryhdb.q diff --git a/config/settings/queryhdb.q b/config/settings/queryhdb.q new file mode 100644 index 000000000..6afa9b323 --- /dev/null +++ b/config/settings/queryhdb.q @@ -0,0 +1,9 @@ +// Bespoke HDB config + +\d .proc +loadprocesscode:1b // whether to load the process specific code defined at ${KDBCODE}/{process type} + +// Server connection details +\d .servers +CONNECTIONS:() // list of connections to make at start up +STARTUP:1b // create connections From 1a3fe7013c7c1292fd8e6b27832b94d57846c14b Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 6 Dec 2022 16:05:55 +0000 Subject: [PATCH 025/114] Add querysort csv --- config/querysort.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 config/querysort.csv diff --git a/config/querysort.csv b/config/querysort.csv new file mode 100644 index 000000000..4170a28c1 --- /dev/null +++ b/config/querysort.csv @@ -0,0 +1,3 @@ +tabname,att,column,sort +default,p,procname,1 +default,,time,1 From c100c7d6d1d939c78e9b540578193123113a88e7 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 6 Dec 2022 16:24:19 +0000 Subject: [PATCH 026/114] Add querytrack enabling to logusage script --- code/handlers/logusage.q | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index ff41a03f7..84e922533 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -2,7 +2,7 @@ // based on logusage.q from code.kx // http://code.kx.com/wsvn/code/contrib/simon/dotz/ -// Modifications : +// Modifications : // usage table is stored in memory // Data is written to file as ASCII text // Added a new LEVEL - LEVEL 0 = nothing; 1=errors only; 2 = + open and queries; 3 = log queries before execution also @@ -27,6 +27,7 @@ suppressalias:@[value;`suppressalias;0b] // whether to suppress the log file al logtimestamp:@[value;`logtimestamp;{[x] {[].proc.cd[]}}] // function to generate the log file timestamp suffix logroll:@[value;`logroll;1b] // whether to automatically roll the log file LEVEL:@[value;`LEVEL;3] // Log level +querytrack:@[value;`querytrack;0b] id:@[value;`id;0j] nextid:{:id+::1} @@ -38,7 +39,7 @@ logh:@[value;`logh;0] write:{ if[logtodisk;@[neg logh;format x;()]]; if[logtomemory; `.usage.usage upsert x]; - ext[x]} + if[querytrack; .ps.publish[`usage;x]]} // extension function to extend the logging e.g. publish the log message ext:{[x]} @@ -51,26 +52,26 @@ flushusage:{[flushtime] delete from `.usage.usage where time<.proc.cp[] - flusht createlog:{[logdir;logname;timestamp;suppressalias] basename:"usage_",(string logname),"_",(string timestamp),".log"; - // Close the current log handle if there is one + // Close the current log handle if there is one if[logh; @[hclose;logh;()]]; // Open the file .lg.o[`usage;"creating usage log file ",lf:logdir,"/",basename]; logh::hopen hsym`$lf; // Create an alias if[not suppressalias; - .proc.createalias[logdir;basename;"usage_",(string logname),".log"]]; + .proc.createalias[logdir;basename;"usage_",(string logname),".log"]]; } -// read in a log file -readlog:{[file] - // Remove leading backtick from symbol columns, convert a and w columns back to integers - update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, - a:"I"$-1 _' a, w:"I"$-1 _' w from +// read in a log file +readlog:{[file] + // Remove leading backtick from symbol columns, convert a and w columns back to integers + update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, + a:"I"$-1 _' a, w:"I"$-1 _' w from // Read in file @[{update "J"$'" " vs' mem from flip (cols .usage.usage)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]} // roll the logs -// inmemorypersist = the number +// inmemorypersist = the number rolllog:{[logdir;logname;timestamp;suppressalias;persisttime] if[logtodisk; createlog[logdir;logname;timestamp;suppressalias]]; flushusage[persisttime]} @@ -101,20 +102,17 @@ p3:{if[ignore; if[0h=type z;if[any first[z]~/:ignorelist; :y@z]]]; p2[x;y;z]} if[enabled; // Create a log file rolllogauto[]; - // If the timer is enabled, and logrolling is set to true, try to log the roll file on a daily basis if[logroll; $[@[value;`.timer.enabled;0b]; [.lg.o[`init;"adding timer function to roll usage logs on a daily schedule starting at ",string `timestamp$(.proc.cd[]+1)+00:00]; .timer.rep[`timestamp$.proc.cd[]+00:00;0Wp;1D;(`.usage.rolllogauto;`);0h;"roll query logs";1b]]; .lg.e[`init;".usage.logroll is set to true, but timer functionality is not loaded - cannot roll usage logs"]]]; - if[flushtime>0; $[@[value;`.timer.enabled;0b]; [.lg.o[`init;"adding timer function to flush in-memory usage logs with interval: ",string flushinterval]; .timer.repeat[.proc.cp[];0Wp;flushinterval;(`.usage.flushusage;flushtime);"flush in memory usage logs"]]; .lg.e[`init;".usage.flushtime is greater than 0, but timer functionality is not loaded - cannot flush in memory tables"]]]; - .z.pw:p0[`pw;.z.pw;;]; .z.po:p1[`po;.z.po;];.z.pc:p1[`pc;.z.pc;]; .z.wo:p1[`wo;.z.wo;];.z.wc:p1[`wc;.z.wc;]; @@ -122,3 +120,6 @@ if[enabled; .z.pg:p2[`pg;.z.pg;];.z.pi:p2[`pi;.z.pi;]; .z.ph:p2[`ph;.z.ph;];.z.pp:p2[`pp;.z.pp;]; .z.ps:p3[`ps;.z.ps;];] +//push .usage.usage table into high level namespace for sub functionality +\d . +if[.usage.querytrack; usage:.usage.usage] From f8e2c594885f6545ffffadafe483c517aac7139f Mon Sep 17 00:00:00 2001 From: David-Mc-Nally Date: Wed, 7 Dec 2022 14:15:14 +0000 Subject: [PATCH 027/114] Addition of Query Gateway --- code/processes/querygateway.q | 640 +++++++++++++++++++++++++++++++++ config/settings/querygateway.q | 24 ++ config/settings/queryhdb.q | 2 +- config/settings/queryrdb.q | 21 +- 4 files changed, 672 insertions(+), 15 deletions(-) create mode 100644 code/processes/querygateway.q create mode 100644 config/settings/querygateway.q diff --git a/code/processes/querygateway.q b/code/processes/querygateway.q new file mode 100644 index 000000000..489944ed2 --- /dev/null +++ b/code/processes/querygateway.q @@ -0,0 +1,640 @@ +// This is an asynchronous gateway. For sync calls, use deferred sync e.g. (neg h)"query";result:h[] +// There is a synchronous method, but it should be avoided unless absolutely required +// e.g. when using a non-native API which doesn't support deferred sync. +// Most of the notes and implementation refer to the asynchronous method. The sync methods cause the gateway to block +// and therefore limit the number of queries that can be serviced at the same time. + +// Queries are routed across heterogenous servers as they become available. Each query can query multiple servers, and the different +// parts of the query will be executed as the required back end server becomes available. +// When all the backend results are available, they will be joined together and returned to the client +// Generally speaking, each back end server will only have one query queued at once. When it returns its result, it will be given +// the next query. +// Queries which fail on the back end are not automatically re-run. It is up to the client to re-submit it if required. + +// There are a couple of calls to be used +// .gw.addserver[handle;servertype] +// is used to add a server of the specified type on the given handle. servertype would be for example `hdb or `rdb +// When a client queries they can use a simple method or a more advanced method +// .gw.asyncexec[query;servertypes(list of symbols)] +// will execute the supplied query on each of the specified server types, and raze the results +// .gw.asyncexecjpt[query;servertypes(list of symbols);joinfunction(lambda);postbackfunction(symbol);timeout(timespan)] +// allows the client to specify how the results are joined, posted back and timed out. +// The join function should be a monadic lambda - the parameter is the list of results +// The postback function is the function which will be called back in the client. It should be a diadic function as both +// the query that was run, and the result are posted back. The postback can be the name of a function or a lambda. The client +// can also post up functions with > 2 parameters, as long as the extra parameters are also sent e.g. +// ({[a;b;query;result] ... };`a;`b) is valid +// To not use a postback function, send up the empty list () +// i.e. the same as a deferred sync call +// The timeout value is used to return an error to the client if the query has been waiting / running too long +// asyncexec is equivalent to asyncexecjpt with joinfunction=raze, postbackfunction=`, timeout=0Wn + +// The synchronous calls are +// .gw.syncexec[query;servertypes(list of symbols)] +// which will parallelise the queries across the backend servers, but block at the front end (so only 1 client query can be processed at a time) +// the results will be razed and returned to the client +// .gw.syncexecj[query;servertypes(list of symbols);joinfunction(lambda)] +// allows the client to specify the join function + +// Error handling +// As default, an error from an asynchronous request is sent back as a string. The format of the response can be altered +// using the formatresponse function. +// each error will be prefixed with the errorprefix (currently "error: ") +// Errors for sync requests should be remain flagged as errors using '.(the current definition of formatresponse). +// Errors will be returned when +// a) the query times out +// b) a back end server returns an error +// c) the join function fails +// d) a back end server fails +// e) the client requests a query against a server type which currently isn't active (this error is returned immediately) +// f) the query is executed successfully but the result is too big to serialize and send back ('limit) +// If postback functions are used, the error string will be posted back within the postback function +// (i.e. it will be packed the same way as a valid result) + +// If the client closes connection before getting results, the back end servers will still continue to execute +// Each of the remaining client queries will be unqueued. +// If a back end server fails, the clients which are waiting for a result will be returned an error. + +// The servers which are available is decided using .gw.availableservers[] function. This returns a dictionary of handle!servertype +// Currently this will return all registered and active handles. If you wish to modify the routing, change this function. Examples +// might be to change it so only servers in the same datacentre or country are used, unless there are none available (i.e. only +// route queries over a WAN if there isn't any other option). To work out where a server is, it needs to report the information +// in the .proc.getattributes[] call. The data will automatically be populated in the .gw.servers table. +// When synchronous calls are used, errors are returned to the client rather than strings. + +// The next query to execute is decided using the .gw.getnextqueryid[] function. Currently this is a simple FIFO queue (.gw.fifo) +// for more sophisticated algorithms e.g. priority queues based on user name, ip address, usage so far, the query that is being run etc. +// set .gw.getnextqueryid to be something else. It should return a 1 row table containing all the query details. + +// server stats are collected in .gw.servers +// client info is in .gw.clients +// query info is in .gw.queryqueue + +// To create a simple homogenous gateway (i.e. all backend servers are the same) you create projections of +// addserver and asyncexec, with the (servertypes) parameter projected to a single server of (for example) `standardserver + +\d .gw + +// if error & sync message, throws an error. Else passes result as normal +// status - 1b=success, 0b=error. sync - 1b=sync, 0b=async +formatresponse:@[value;`.gw.formatresponse;{{[status;sync;result]$[not[status]and sync;'result;result]}}]; +synccallsallowed:@[value;`.gw.synccallsallowed; 0b] // whether synchronous calls are allowed +querykeeptime:@[value;`.gw.querykeeptime; 0D00:30] // the time to keep queries in the +errorprefix:@[value;`.gw.errorprefix; "error: "] // the prefix for clients to look for in error strings +permissioned:@[value;`.gw.permissioned; 0b] // should the gateway permission queries before the permissions script does +clearinactivetime:@[value;`.gw.clearinactivetime; 0D01:00] // the time to store data on inactive handles + +readonly:@[value;`.readonly.enabled;0b] +valp:$[`.pm.valp ~ key `.pm.valp;.pm.valp;.gw.readonly;{value parse x};value] +val:$[`.pm.val ~ key `.pm.val;.pm.val;.gw.readonly;reval;eval] + +eod:0b +seteod:{[b] .lg.o[`eod;".gw.eod set to ",string b]; eod::b;} // called by wdb.q during EOD +checkeod:{[IDS].gw.eod&1 returntime+age} + +// scheduling function to get the next query to execute. Need to ensure we avoid starvation +// possibilities : +// fifo +// low utilisation (handles which have used it least), +// low utilisation last x minutes +// handle longest wait (the handle which has been waiting the longest without any query being serviced) +// low query count (handle with least queries run) +// low query count last x minutes +fifo:{1 sublist select from canberun[] where time=min time} +getnextqueryid:fifo +getnextquery:{ + qid:getnextqueryid[]; + if[0=count qid; :()]; + update submittime:.proc.cp[]^submittime from `.gw.queryqueue where queryid in qid`queryid; + qid} + +// finish a query +// delete the temp results +// update the status, set the return time +// reset the serverhandle +finishquery:{[qid;err;serverh] + deleteresult[qid]; + update error:err,returntime:.proc.cp[] from `.gw.queryqueue where queryid in qid; + setserverstate[serverh;0b]; + } + +// Get a list of pending and running queries +getqueue:{select queryid,time,clienth,query,servertype,status:?[null submittime;`pending;`running],submittime from .gw.queryqueue where null returntime} + +// manage the result set dictionaries +addemptyresult:{[queryid; clienth; servertypes] results[queryid]:(clienth;servertypes!(count servertypes,:())#enlist(0Ni;(::);0b))} +addservertoquery:{[queryid;servertype;serverh] .[`.gw.results;(queryid;1);{.[x;(y 0;0);:;y 1]};(servertype;serverh)]} +deleteresult:{[queryid] .gw.results : (queryid,()) _ .gw.results} + +// add a result coming back from a server +addserverresult:{[queryid;results] + serverid:first exec serverid from .gw.servers where active, handle=.z.w; + if[queryid in key .gw.results; + .[`.gw.results;(queryid;1;.gw.results[queryid;1;;0]?serverid;1);:;results]; + .[`.gw.results;(queryid;1;.gw.results[queryid;1;;0]?serverid;2);:;1b] + ]; + setserverstate[.z.w;0b]; + runnextquery[]; + checkresults[queryid]} +// handle an error coming back from the server +addservererror:{[queryid;error] + // propagate the error to the client + sendclientreply[queryid;.gw.errorprefix,error;0b]; + setserverstate[.z.w;0b]; + runnextquery[]; + // finish the query + finishquery[queryid;1b;0Ni]; + } +// check if all results are in. If so, send the results to the client +checkresults:{[queryid] + if[all value (r:.gw.results[queryid])[1;;2]; + // get the rest of the detail from the query table + querydetails:queryqueue[queryid]; + // apply the join function to the results + res:@[{(0b;$[10h=type x;value(x;y); x @ y])}[querydetails[`join]];value r[1;;1];{(1b;.gw.errorprefix,"failed to apply join function to result sets: ",x)}]; + // send the results back to the client. + sendclientreply[queryid;last res;not res 0]; + // finish the query + finishquery[queryid;res 0;0Ni]];} + +// build and send a response to go to the client +// if the postback function is defined, then wrap the result in that, and also send back the original query +sendclientreply:{[queryid;result;status] + querydetails:queryqueue[queryid]; + // if query has already been sent an error, don't send another one + if[querydetails`error; :()]; + tosend:$[()~querydetails[`postback]; + result; + querydetails[`postback],enlist[querydetails`query],enlist result]; + $[querydetails`sync; + // return sync response + @[-30!;(querydetails`clienth;not status;$[status;.gw.formatresponse[1b;1b;result];result]);{.lg.o[`syncexec;x]}]; + @[neg querydetails`clienth;.gw.formatresponse[status;0b;tosend];()]]; + }; + +// execute a query on the server. Catch the error, propagate back +serverexecute:{[queryid;query] + res:@[{(0b;value x)};query;{(1b;"failed to run query on server ",(string .z.h),":",(string system"p"),": ",x)}]; + // send back the result, in an error trap + @[neg .z.w; $[res 0; (`.gw.addservererror;queryid;res 1); (`.gw.addserverresult;queryid;res 1)]; + // if we fail to send the result back it might be something IPC related, e.g. limit error, so try just sending back an error message + {@[neg .z.w;(`.gw.addservererror;x;"failed to return query from server ",(string .z.h),":",(string system"p"),": ",y);()]}[queryid]];} +// send a query to a server +sendquerytoserver:{[queryid;query;serverh] + (neg serverh,:())@\:(serverexecute;queryid;query); + setserverstate[serverh;1b];} + +// handle is closed by a server +removeserverhandle:{[serverh] + if[null servertype:first exec servertype from .gw.servers where handle=serverh; :()]; + if[null serverid:first exec serverid from .gw.servers where handle=serverh; :()]; + + // get the list of effected query ids + + // 1) queries sent to this server but no reply back yet + qids:where {[res;id] any not res[1;where id=res[1;;0];2]}[;serverid] each results; + // propagate an error back to each client + sendclientreply[;.gw.errorprefix,"backend ",string[servertype]," server handling query closed the connection";0b] each qids; + finishquery[qids;1b;serverh]; + + // 2) queries partially run + waiting for this server + activeServerIDs:exec serverid from .gw.servers where active, handle<>serverh; + activeServerTypes:distinct exec servertype from .gw.servers where active, handle<>serverh; + + qids2:where {[res;id;aIDs;aTypes] + s:where not res[1;;2]; + $[11h=type s; not all s in aTypes; not all any each s in\: aIDs] + }[;serverid;activeServerIDs;activeServerTypes] each results _ 0Ni; + sendclientreply[;.gw.errorprefix,"backend ",string[servertype]," server for running query closed the connection";0b] each qids2; + finishquery[qids2;1b;serverh]; + + // 3) queries not yet run + waiting for this server + qids3:exec queryid from .gw.queryqueue where null submittime, not `boolean${$[11h=type z; all z in x; all any each z in\: y]}[activeServerTypes;activeServerIDs] each servertype; + // propagate an error back to each client + sendclientreply[;.gw.errorprefix,"backend ",string[servertype]," server for queued query closed the connection";0b] each qids3; + finishquery[qids3;1b;serverh]; + + // mark the server as inactive + update handle:0Ni, active:0b, disconnecttime:.proc.cp[] from `.gw.servers where handle=serverh; + + runnextquery[]; + } + +// clear out long time inactive servers +removeinactive:{[inactivity] + delete from `.gw.servers where not active,disconnecttime<.proc.cp[]-inactivity} + +// timeout queries +checktimeout:{ + qids:exec queryid from .gw.queryqueue where not timeout=0Wn,.proc.cp[] > time+timeout,null returntime; + // propagate a timeout error to each client + if[count qids; + sendclientreply[;.gw.errorprefix,"query has exceeded specified timeout value";0b] each qids; + finishquery[qids;1b;0Ni]]; + } + + + + +/- NEED TO FILTER ON PREFERENCES FIRST + +/- initial common bit of functionality for filtering on servers +getserversinitial:{[req;att] + + if[0=count req; :([]serverid:enlist key att)]; + + /- check if all servers report all the requirements - drop any that don't + att:(where all each (key req) in/: key each att)#att; + + if[not count att;'"getservers: no servers report all requested attributes"]; + + /- calculate where each of the requirements is in each of the attribute sets + s:update serverid:key att from value req in'/: (key req)#/:att; + + /- calculate how many requirements are satisfied by each server + /- the requirments satisfied is the minimum of each + /- resort the table by this + s:s idesc value min each sum each' `serverid xkey s; + + /- split the servers into groups - those with the same attributes are in the same group + /- we only need to query one of these + s:`serverid xkey 0!(key req) xgroup s; + + s} + +/- given a dictionary of requirements and a list of attribute dictionaries +/- work out which servers we need to hit to satisfy each requirement +/- each requirement only has to be satisfied once, i.e. requirements are treated independently +getserversindependent:{[req;att;besteffort] + + if[0=count req; :([]serverid:enlist key att)]; + + s:getserversinitial[req;att]; + + /- we want to calculate which rows have not already been fully satisfied + /- if the matched value in a row has already been matched, then it is useless + filter:(value s)¬ -1 _ (0b&(value s) enlist 0),maxs value s; + + /- work out whether the requirement is completely filled + alldone:1+first where all each all each' maxs value s; + + if[(null alldone) and not besteffort; + '"getserversindependent: cannot satisfy query as not all attributes can be matched"]; + + /- use the filter to remove any rows which don't add value + s:1!(0!s) w:where any each any each' filter; + + /- map each server id group to each of the attributes that it has available + /- if you want overlaps, remove the &filter w from the end of this bit of code + (key s)!{(key x)!(value x)@'where each y key x}[req]each value s&filter w} + +// execute an asynchronous query +asyncexecjpts:{[query;servertype;joinfunction;postback;timeout;sync] + // Check correct function called + if[sync<>.gw.call .z.w; + // if asyncexec used with sync request, signal an error. + // if syncexec used with async request, send async response using neg .z.w + @[neg .z.w;.gw.formatresponse[0b;not sync;"Incorrect function used: ",$[sync;"syncexec";"asyncexec"]];()]; + :(); + ]; + if[.gw.permissioned; + if[not .pm.allowed[.z.u;query]; + @[neg .z.w;.gw.formatresponse[0b;sync;"User is not permissioned to run this query from the gateway"];()]; + :(); + ]; + ]; + query:({[u;q]$[`.pm.execas ~ key `.pm.execas;value (`.pm.execas; q; u);`.pm.valp ~ key `.pm.valp; .pm.valp q; value q]}; .z.u; query); + /- if sync calls are allowed disable async calls to avoid query conflicts + $[.gw.synccallsallowed and .z.K<3.6; + errstr:.gw.errorprefix,"only synchronous calls are allowed"; + [errstr:""; + if[99h<>type servertype; + // its a list of servertypes e.g. `rdb`hdb + servertype:distinct servertype,(); + errcheck:@[getserverids;servertype;{.gw.errorprefix,x}]; + if[10h=type errcheck; errstr:errcheck]; + queryattributes:()!(); + ]; + if[99h=type servertype; + // its a dictionary of attributes + queryattributes:servertype; + res:@[getserverids;queryattributes;{.gw.errorprefix,"getserverids: failed with error - ",x}]; + if[10h=type res; errstr:res]; + if[10h<>type res; if[0=count raze res; errstr:.gw.errorprefix,"no servers match given attributes"]]; + servertype:res; + ]; + ] + ]; + // error has been hit + if[count errstr; + @[neg .z.w;.gw.formatresponse[0b;sync;$[()~postback;errstr;$[-11h=type postback;enlist postback;postback],enlist[query],enlist errstr]];()]; + :()]; + + addquerytimeout[query;servertype;queryattributes;joinfunction;postback;timeout;sync]; + runnextquery[]; + }; + +asyncexecjpt:asyncexecjpts[;;;;;0b] +asyncexec:asyncexecjpt[;;raze;();0Wn] + +// execute a synchronous query +syncexecjpre36:{[query;servertype;joinfunction] + // Check correct function called + if[(.z.w<>0)&(.z.w in key .gw.call)¬ .gw.call .z.w; + @[neg .z.w;.gw.formatresponse[0b;0b;"Incorrect function used: asyncexec"];()]; + :(); + ]; + if[not[.gw.synccallsallowed] and .z.K<3.6;.gw.formatresponse[0b;1b;"synchronous calls are not allowed"]]; + // check if the gateway allows the query to be called + if[.gw.permissioned; + if[not .pm.allowed [.z.u;query]; + .gw.formatresponse[0b;1b;"User is not permissioned to run this query from the gateway"]; + ]; + ]; + // check if we have all the servers active + serverids:@[getserverids;servertype;{.gw.formatresponse[0b;1b;.gw.errorprefix,x]}]; + // check if gateway in eod reload phase + if[checkeod[serverids];.gw.formatresponse[0b;1b;"unable to query multiple servers during eod reload"]]; + // get the list of handles + tab:availableserverstable[0b]; + handles:(exec serverid!handle from tab)first each (exec serverid from tab) inter/: serverids; + setserverstate[handles;1b]; + start:.z.p; + query:({[u;q]$[`.pm.execas ~ key `.pm.execas; value(`.pm.execas; q; u);`.pm.valp ~ key `.pm.valp; .pm.valp q; value q]}; .z.u; query); + // to allow parallel execution, send an async query up each handle, then block and wait for the results + (neg handles)@\:({@[neg .z.w;@[{(1b;.z.p;value x)};x;{(0b;.z.p;x)}];{@[neg .z.w;(0b;.z.p;x);()]}]};query); + // flush + (neg handles)@\:(::); + // block and wait for the results + res:handles@\:(::); + // update the usage data + update inuse:0b,usage:usage+(handles!res[;1] - start)[handle] from `.gw.servers where handle in handles; + // check if there are any errors in the returned results + :$[all res[;0]; + // no errors - join the results + [s:@[{(1b;x y)}joinfunction;res[;2];{(0b;"failed to apply supplied join function to results: ",x)}]; + .gw.formatresponse[s 0;1b;s 1]]; + [failed:where not res[;0]; + .gw.formatresponse[0b;1b;"queries failed on server(s) ",(", " sv string exec servertype from servers where handle in handles failed),". Error(s) were ","; " sv res[failed][;2]]] + ]; + }; + +syncexecjt:{[query;servertype;joinfunction;timeout] + // can only be used on 3.6 + + // use async call back function, flag it as sync + // doesn't make sense to allow specification of a callback for sync requests + asyncexecjpts[query;servertype;joinfunction;();timeout;1b]; + // defer response + @[-30!;(::);()]; + }; + +$[.z.K < 3.6; + [syncexecj:syncexecjpre36; + syncexec:syncexecjpre36[;;raze]]; + [syncexecj:syncexecjt[;;;0Wn]; + syncexec:syncexecjt[;;raze;0Wn]]]; + +// run a query +runquery:{[] + // check if there is something to run + if[count torun:getnextquery[]; + if[not checkeod torun`servertype; + torun:first torun; + // if it isn't already in the result dict, add it + if[not torun[`queryid] in key results; + addemptyresult[torun`queryid;torun`clienth;torun`servertype]; + ]; + // update the results dictionary and send off the queries + // get the handles to run on + avail:availableserverstable[1b]; + IDs:$[11h=type torun`available; + (exec first serverid by servertype from avail)[torun`available]; + first each (exec serverid from avail) inter/: torun`available]; + handles:avail[([]serverid:IDs);`handle]; + // update the results dictionary + addservertoquery[torun`queryid;torun`available;IDs]; + // send off the queries + sendquerytoserver[torun`queryid;torun`query;handles]; + ]; + ]; + }; + +runnextquery:runquery + + +// when a new connection is opened, add client details +po:{addclientdetails[x]} + +// called when a handle is closed +pc:{ + removeclienthandle[x]; + removeserverhandle[x];} + +pgs:{.gw.call,:enlist[x]!enlist y}; + +// override message handlers +.z.pc:{x@y;.gw.pc[y]}@[value;`.z.pc;{{[x]}}]; +.z.po:{x@y;.gw.po[y]}@[value;`.z.po;{{[x]}}]; +.z.pg:{.gw.pgs[.z.w;1b];x@y}@[value;`.z.pg;{{[x]}}]; +.z.ps:{.gw.pgs[.z.w;0b];x@y}@[value;`.z.ps;{{[x]}}]; +// only wrap .z.ws if it is already defined +if[@[{value x;1b};`.z.ws;{0b}];.z.ws:{.gw.pgs[.z.w;0b];x@y}.z.ws]; + +// START UP +// initialise connections +.servers.startup[] + + /-check if the gateway has connected to discovery process, block the process until a connection is established +while[0 = count .servers.getservers[`proctype;`discovery;()!();0b;1b]; + /-while no connected make the process sleep for X seconds and then run the subscribe function again + .os.sleep[5]; + /-run the servers startup code again (to make connection to discovery) + .servers.startup[]; + .servers.retrydiscovery[]] + +// add servers from the standard connections table +addserversfromconnectiontable:{ + {.gw.addserverattr'[x`w;x`proctype;x`attributes]}[select w,proctype,attributes from .servers.SERVERS where ((proctype in x) or x~`ALL),not w in ((0;0Ni),exec handle from .gw.servers where active)];} + +// When new connections come in from the discovery service, try to reconnect +.servers.addprocscustom:{[connectiontab;procs] + // retry connections + .servers.retry[]; + // add any new connections + .gw.addserversfromconnectiontable[.servers.CONNECTIONS]; + // See if any queries can now be run + runnextquery[]} + +addserversfromconnectiontable[.servers.CONNECTIONS] + +// Join active .gw.servers to .servers.SERVERS table +activeservers:{lj[select from .gw.servers where active;`handle xcol `w xkey .servers.SERVERS]} + +\d . + +// functions called by end-of-day processes + +reloadstart:{ + .lg.o[`reload;"reload start called"]; + /- set eod variable to active/true + .gw.seteod[1b]; + /- extract ids of queries not yet returned + qids:exec queryid from .gw.queryqueue where 1 Date: Fri, 9 Dec 2022 14:42:23 +0000 Subject: [PATCH 028/114] Modified connections made by queryfeed --- code/processes/queryfeed.q | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index 14223d109..6db8f2de6 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -1,13 +1,18 @@ +// queryfeed proc script - subs to .usage.usage tables and publishes to query tickerplant + +// add connections to all procs for query tracking to be enabled +.servers.CONNECTIONS:.servers.CONNECTIONS,exec distinct proctype from (" SS ";enlist csv) 0: hsym `$getenv `TORQPROCESSES where procname in subprocs; + us:@[value;`us;([]querytime:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$();procname:`symbol$;status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())]; -upd:{[t;x] if [t in `usage; `us insert x]}; +upd:{[t;x] if [t in `.usage.usage; `us insert x]}; .servers.startup[]; start_sub:{[subprocs] - hds:(),exec w from .servers.SERVERS where proctype in subprocs; + hds:(),exec w from .servers.SERVERS where procname in subprocs; { .lg.o[`startsub;"subscribing to ", string first exec procname from .servers.SERVERS where w=x]; - x(`.u.sub;`usage;`); + x(`.usage.querysub;`.usage.usage;`); .lg.o[`completesub;"subscribed"]; }each hds; }; From 28503aec4e276a9826f3a79a9a77e0b64adb65cf Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 9 Dec 2022 14:51:15 +0000 Subject: [PATCH 029/114] Added config csv to enable proc query-tracking --- config/querytrack.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 config/querytrack.csv diff --git a/config/querytrack.csv b/config/querytrack.csv new file mode 100644 index 000000000..0032cedcc --- /dev/null +++ b/config/querytrack.csv @@ -0,0 +1,6 @@ +rdb1 +hdb1 +rdb2 +hdb2 +gateway1 + From d98f3ac418ca96b5eb10c31100b2ee134fc34a73 Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 9 Dec 2022 15:03:04 +0000 Subject: [PATCH 030/114] Config listing procs for queryfeed to make connections --- config/settings/queryfeed.q | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/settings/queryfeed.q b/config/settings/queryfeed.q index d6cb85e4a..849919545 100644 --- a/config/settings/queryfeed.q +++ b/config/settings/queryfeed.q @@ -1,8 +1,7 @@ \d .servers enabled:1b -CONNECTIONS:`qtp`rdb`hdb // Feedhandler connects to the tickerplant +CONNECTIONS:`qtp; // Feedhandler connects to the query-tickerplant HOPENTIMEOUT:30000 \d . -subprocs:`rdb`hdb -reloadenabled:1b +subprocs:"S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.csv"; // List of procs for query-tickerplant to subscribe to From d7e8c07220805a06b7929f3a143e7ee564ea6b94 Mon Sep 17 00:00:00 2001 From: acreehay Date: Fri, 9 Dec 2022 15:22:08 +0000 Subject: [PATCH 031/114] Query tracking enabled and usage published to queryfeed --- code/handlers/logusage.q | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index 84e922533..ec8b3f6a4 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -2,7 +2,7 @@ // based on logusage.q from code.kx // http://code.kx.com/wsvn/code/contrib/simon/dotz/ -// Modifications : +// Modifications : // usage table is stored in memory // Data is written to file as ASCII text // Added a new LEVEL - LEVEL 0 = nothing; 1=errors only; 2 = + open and queries; 3 = log queries before execution also @@ -10,7 +10,7 @@ \d .usage // table to store usage info -usage:@[value;`usage;([]time:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$(); procname:`symbol$(); status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())] +usage:@[value;`usage;([]time:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$(); procname:`symbol$(); status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())] // Check if the process has been initialised correctly if[not @[value;`.proc.loaded;0b]; '"environment is not initialised correctly to load this script"] @@ -27,7 +27,10 @@ suppressalias:@[value;`suppressalias;0b] // whether to suppress the log file al logtimestamp:@[value;`logtimestamp;{[x] {[].proc.cd[]}}] // function to generate the log file timestamp suffix logroll:@[value;`logroll;1b] // whether to automatically roll the log file LEVEL:@[value;`LEVEL;3] // Log level -querytrack:@[value;`querytrack;0b] +querytrack:@[value;`querytrack;0b] // whether query tracking is enabled by default + +// enable query tracking for proc if procname included in csv config file +querytrack:$[.proc.procname in "S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.csv";1b;0b] id:@[value;`id;0j] nextid:{:id+::1} @@ -39,7 +42,11 @@ logh:@[value;`logh;0] write:{ if[logtodisk;@[neg logh;format x;()]]; if[logtomemory; `.usage.usage upsert x]; - if[querytrack; .ps.publish[`usage;x]]} + if[querytrack; .ps.publish[`.usage.usage;x]]; + ext[x]} + +// custom sub function to add usage table to subscribable tables if query tracking enabled +querysub:{if[querytrack; .stpps.t:`.usage.usage,.stpps.t]; .u.sub[x;y]} // extension function to extend the logging e.g. publish the log message ext:{[x]} @@ -52,26 +59,26 @@ flushusage:{[flushtime] delete from `.usage.usage where time<.proc.cp[] - flusht createlog:{[logdir;logname;timestamp;suppressalias] basename:"usage_",(string logname),"_",(string timestamp),".log"; - // Close the current log handle if there is one + // Close the current log handle if there is one if[logh; @[hclose;logh;()]]; // Open the file .lg.o[`usage;"creating usage log file ",lf:logdir,"/",basename]; logh::hopen hsym`$lf; // Create an alias if[not suppressalias; - .proc.createalias[logdir;basename;"usage_",(string logname),".log"]]; + .proc.createalias[logdir;basename;"usage_",(string logname),".log"]]; } -// read in a log file -readlog:{[file] - // Remove leading backtick from symbol columns, convert a and w columns back to integers - update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, - a:"I"$-1 _' a, w:"I"$-1 _' w from +// read in a log file +readlog:{[file] + // Remove leading backtick from symbol columns, convert a and w columns back to integers + update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, + a:"I"$-1 _' a, w:"I"$-1 _' w from // Read in file - @[{update "J"$'" " vs' mem from flip (cols .usage.usage)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]} + @[{update "J"$'" " vs' mem from flip (cols .usage.usage)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]} // roll the logs -// inmemorypersist = the number +// inmemorypersist = the number rolllog:{[logdir;logname;timestamp;suppressalias;persisttime] if[logtodisk; createlog[logdir;logname;timestamp;suppressalias]]; flushusage[persisttime]} @@ -102,17 +109,20 @@ p3:{if[ignore; if[0h=type z;if[any first[z]~/:ignorelist; :y@z]]]; p2[x;y;z]} if[enabled; // Create a log file rolllogauto[]; + // If the timer is enabled, and logrolling is set to true, try to log the roll file on a daily basis if[logroll; $[@[value;`.timer.enabled;0b]; [.lg.o[`init;"adding timer function to roll usage logs on a daily schedule starting at ",string `timestamp$(.proc.cd[]+1)+00:00]; .timer.rep[`timestamp$.proc.cd[]+00:00;0Wp;1D;(`.usage.rolllogauto;`);0h;"roll query logs";1b]]; .lg.e[`init;".usage.logroll is set to true, but timer functionality is not loaded - cannot roll usage logs"]]]; + if[flushtime>0; $[@[value;`.timer.enabled;0b]; [.lg.o[`init;"adding timer function to flush in-memory usage logs with interval: ",string flushinterval]; .timer.repeat[.proc.cp[];0Wp;flushinterval;(`.usage.flushusage;flushtime);"flush in memory usage logs"]]; .lg.e[`init;".usage.flushtime is greater than 0, but timer functionality is not loaded - cannot flush in memory tables"]]]; + .z.pw:p0[`pw;.z.pw;;]; .z.po:p1[`po;.z.po;];.z.pc:p1[`pc;.z.pc;]; .z.wo:p1[`wo;.z.wo;];.z.wc:p1[`wc;.z.wc;]; @@ -120,6 +130,3 @@ if[enabled; .z.pg:p2[`pg;.z.pg;];.z.pi:p2[`pi;.z.pi;]; .z.ph:p2[`ph;.z.ph;];.z.pp:p2[`pp;.z.pp;]; .z.ps:p3[`ps;.z.ps;];] -//push .usage.usage table into high level namespace for sub functionality -\d . -if[.usage.querytrack; usage:.usage.usage] From 0acca9fbe1255db45b528eb51060387b6ce37224 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Fri, 9 Dec 2022 16:51:22 +0000 Subject: [PATCH 032/114] Add Query Logging Management documentation --- QueryManagement.md | 56 ++++++++++++++++++++++++++++++++++++++ torq-qlm_architecture.PNG | Bin 0 -> 12388 bytes 2 files changed, 56 insertions(+) create mode 100644 QueryManagement.md create mode 100644 torq-qlm_architecture.PNG diff --git a/QueryManagement.md b/QueryManagement.md new file mode 100644 index 000000000..69dcc1369 --- /dev/null +++ b/QueryManagement.md @@ -0,0 +1,56 @@ + + +Functionality Overview +====================== + +Query Logging Management is an addition to TorQ to enhance the current query logging system. The aim for this tool is to have access to information about queries that are sent to specific processes throughout the day, allowing for query analysis to be carried. + +For each query executed we want access to: + +- The time of the query +- The amount of time the query took to run +- Username of the person running the query +- IP address of the person running the query +- Host of the process that was being queried +- Name of the process that was being queried +- Process type of the process that was being queried +- The query that was being executed including any parameters + +This query logging functionality can be enabled or disabled for specific processes using variables within TorQ config files. + +Architecture +============ + +The architecture of the Query Logging Management framework is shown in the following diagram: + +![QueryLoggingManagementArchitecture](graphics/torq-qlm_architecture.PNG) + +Processes +======== + +Query Feed +---------- + +If enabled within a TorQ process, the Query Feed process subscribes to updates from the .usage.usage table that is defined in the logusage.q handlers script from all enabled TorQ processes. + +Once a connection has been set up to our Query Tickerplant, the Query feed sends a message to execute the .u.upd function to inesrt this collected .usage.usage table into a usage table. + +Query Tickerplant +----------------- + +The Query Tickerplant process receives updates from the Query Feed process regarding the usage table and publishes it on to any subscribing process, operating the same way as a standard tickerplant. + +Query RDB +--------- + +The Query RDB works like a normal RDB process, receiving usage table messages from the Tickerplant and storing this in memory in order to be queried. At the end of day, the usage table is saved down onto disk to be loaded into the HDB. + +Query HDB +--------- + +The Query HDB loads in historical usage data from disk in order for long term query information to be queried. + +Query Gateway +------------- + +The Query Gateway process subscribes to our specific Query RDB and HDB processes to load balance queries and allow for access to query information involving both historical and real time data. The query gateway process also contains a number of analytics functions to return specific aggregations from the usage table (i.e how many queries were executed for each process on a specific day). diff --git a/torq-qlm_architecture.PNG b/torq-qlm_architecture.PNG new file mode 100644 index 0000000000000000000000000000000000000000..976a418114796a7e7d2cf0bbf31575843cbfb8bd GIT binary patch literal 12388 zcmdUWc|6qbyZ5wFqe)paM5RzdGM32F*h==jY@shBWKEV(F@z?iq^u1gyUCgiH5ir3 zAo~`wjb)k`%h;a#L*H}G@0{m(Ug!0C&i4G#Yx>;F^|`M5zSj5k5oM^ajesNJFc=JR z@q&gi47RNl2HSpfHwzdcV0L{5Kihnawb8J`ZvvCxVW;zXz4KrymTlF37kJ*|dEv4T z494LL{o6L+B;*f+NiAR0IB)82JJav}z`8wU+^e<+GpHe6aC8QK)LpCRQ&N7{4+WRT znPb~oamDfUd%cg&mmJF2ICb&!S1keAz|-#!4a?@+p#P#&%5Xyu*(1AGI{I*l{T@xiPhJ?+M#>i+3q{E}jt_S8i{toiJB+`ne$ewRvm4!8 zHKn#x|JjN(d};ly*Wrr+EtW1{P=3F#DVO-^H9jtHD5(9!GgibBFUp0tEziH&y?q#; zC_HW56VsV;$p@R2`8&JWs`;n10Iq*2DCmB5mXnZDg5QYOc&jyj@*r`aD+;lnz=<0w z;}7iSB zMIm+*5(X@7tKX_t6^Ex>vZKH8aml637v36pH7Sa0GDuxXS)U+r_(=vgG=0m=LAPaE zC$fZf;mBMIe@LA`#mRCd>V44+rJv^g=LbB6!;xc|4eP||(_$x9TK`;Q9gXjLQ(Pe`5+D0V8 z(IrvnIfzMdthv;y&=s0KwZCx1Inb@Fl*gvDj9zoT*)Fsi3;4gB5+l9w*W}>BY-cUK zoZ6S?%;k6HL}`7#{!wn+jg7^PNfQ|rxhxmsz05@al~38L6&W`w_QSu5G-Ks3AtzFA zL5oir9PL-p#W(D+Acj&p>q)ODk3NfWUwJTVl3RfQ``%@qy@)^Vbdy7t`=+=~bS5egFN}vf}ejkf2%50E1#HuX! zEX|l`!#6KW_q!A+@319q;#e2GMysN|R{IlBm6CH&)F4+nL5~l$?ct(qn27x(Bv&Sy zKR!hlEG6 zLN=CYNcLx4kWW>))jdo8aCjk|u)3P7x7LC^K5fm6PDr_Ag7sUzxfnq=Ta??ko4|`3 z#ST`#`v<=;yWG-T=PX&3;iYz;c$>K@H2=?jK##j8uQo{Z_U#6}p;9T;Rt2$oeN+VU z67Iyb{dlWI^|W>3Nz~hrxeWX1RV98_Y}cpZm|1w|jU}&8Q+tF`LddWSUS`GilDUp|p?f4=ea9irxuT|&x#3@lZ(XwdDkbbu|1~` zh0H9j1>N~j8eLR0djhpL2CbgwHxra&;XuFn$W>0Yt(H|mWoGZ>Es@#%UaygL=@iPI zLr&KW_)9W7n;c$D-qMcDz1;6;$%Ry$TeGR1aephn_Ec zD0*jNefx_L3x&J8Mem{2vrVtj4LJ<6x5tQv6}N?PYU1%e`K_n#+lsf*BKDo#E*gzi zANMuhorn7Fa+9YS^Rxa)>0PushhnfDzocX&RxA}&Z>2=xfg>(XaJjixbLlrmgs@ts z;N7{9aZw)S9kmDDujG^{s`rYaj%9S}4xhy3sXZvHR5V#qR!ah7+!ij$?Q&JaG+0q~ zIL2HgCc1bU+fEbDkyC~k> zc)ait8#W@{^Wv!$K!X>KXc&sK^lLc8@n?~?=z)4Oq(^zV`YxVZTD~({hXHe%!*P}i zp4%LLYT@x00L5q2UP%Q!>sv#N7>~!7GT|EORJQ%XqJUVdZK!o7eXP&hC3!wDn;dn? z`Un`-9Er0Wjd85kLB2J@h+Scn@kvIRrBZj@zpN=N4|YM*(M3LmgLL!;&tV}rLd28B zCSX4B1vz|?EuD)Csrv)07~-%HLtt5CDvz0;t%OPIXy8BW1$6eAJBuB_vY3G35bHY! zGt((obr@ee@>^TmnpFjvqdN~hjX;N;wdN2{^WWwG#N;r-oX)P&8Tnd0IWaCcsPXl9 zq2W>R&W0uly3>>|ID8##{D)%&JbN&SmR6|n1yX;F=U|jf#3^At-_^-%S^7 zpZLx{O5D5(BB7KoIi{Z{ld>~CO(>!Ao-7x#58S6uK@FK@mIs!U-P_}ubNYMWjWwI8yq2=5hyqKdgP|2zXsm^)h})F zw=n5d;Hiv#sUg;xIBHS>7^nsV-la!6@#RLRN`ck{*a=qSDR&NV6eW|67ydW^^qI(f zcZ;N~$_%I+_V5d@U2UDQU+rTIJuSI$jPK1v@~2-|HV@jSuio)A9IgI%vsw_USu7g* zUgEJYA%Dr(lYdg-hH+xLs;>mkI;?U|w<+Fg`T^hV_ibfddki_0S`MzSG1KO?xi0|6?5DV9vTb*RmH; zlc--=h+~9=n1udzeiO)wyC-o~fgUDpyVPBn)^dcu0v<{U{?$$l^;7MC{{P-_ zz}?3i1>tjJUzOsxg%lS*&WMKosMA*V8)D_f>Io4?o)@*ugr#b4*e*Wy5DM7WUqNr) zpg(W?Hg-1YhRLu`!hSg7Eb!cg;b@a1uU`$1pTvplUuhG6IKq8pt%m8&w5%}mupl4; zXWtHFjpwO~IBy-yyFP5zy(saVgWTrUd)i|hB(YhsBPS&WRFjH?#>$rl&4>~fe^zq9 z^~95@?k68~b6sIPN=wZSNm40N$-WUYn77_lc|C8SV?LQ)&J0a}KS9@tfq-N4NgNg2 zXNo!bG|M;#%vRG~r($tuANM-6n0qvV00A*9c?+K-Q2M*2mM#~Mg`Wr!h&iev7jn*| zWnzRn=VyjtdKCNvv#*PDBNrlLz@C=>HFW`1fk=JGxJ2q&Dxl_DcI>AWOC`^?svTrKSkN z@k*0azu@elpmeXl4aehe!>Rcl>KAb{t-HU{3))n@8rl!??!$+`srAogq(uUk(gjU< z)8D)c+_I#17NLv!3XJKby4u^nKaqjua^mo0P=wvuYB?EqjPOME5^Y~- z?Yiii(XQ5XieY%gz_R+%bU;N!FV9A$m-AM=|oguyQJ^Ch5dZ#y@V)(S+-M)Fb2l?NpH5@agSB8iN4DACS!_ewpr3ppq7J+&= zPR-9avR2Uxmwd|?_?>{@t4@30^p82tgWr`9)-?B4e!sWr7iuP)i=-)*7+qhv41L&; ztjtr8(Y5$8D+bzL$y2FR&Z<{&g`5{!wN+Q7tUZG^{q)SD^4FYYv3*z;UpJ7^LLiO( zN`};e$2JN8iY5+XSzevTZ#$Ghxq}izy>uKLWc1qNC+%SU!23Tm!l)s+kQ8Y+;zAgD zhhkReH&aZY1VZf=w3G|^hy|kBS(6Zm^;@%PUi?=ylT?B zMYc5Vo=y?=f6dMc$f|>|<;;k+H7l4ez_Apy@Y`GrF=~zbY8O^)@hP`v3|1;&69DG& zeSqH43K1tVW`)3>)m-Ag-Xn%Oi~u62VY6Tp;2@3xNUR?SywkVR2%A+G3#$Srum=!d z1Pix;LlFdPUdo&0QRYQlVf}#iPTxb?+=+!#yI=AHsbS!yvFIHfU}6@W(01lX>PgfZ z!C-4CIGP}^=)G)G+9Y?+{09rbCx-uDytS})YS|V))3ay{js@hjcXs~0g?UJ79`63< zcH{sfk8EYyrkv@m7EBA;#SsomwH2^bz3QOXsgi>!B=(oV(B%+sm`k8+wRdbvVak5? zx9GJyH-I$Tz2eCik3e^I_&~~D5_RvU?C2NU`hh}Mt4q}$DDSIP&1aI!i>FCn?cGMJ zJU?HSkK5bQ9MZijiIiJpzyNjE4oJrc`)YqXSq3%Do(}nTt@8G=X;+o^$IB;W(VQAyYep&OYd(H#ecQzrNOfBib;u&|PuS)YFSkX+bw$~l zv{`P?0j%89-{|OmkPLHBMb^=yF6vKedqc?p2;|>aHv@UovziSR*~6fe(4_F+WDI0w z;eL%8DKE`qDYEaK1Ssv}omHMeMoA*%kJUF;zi^(M43>f=ox zz{Pp=%C0_r%ay3|Ft0RgJSO(+KxYDS>kt8p=BlOH953|nX6)?L0BVIf%;Jo?h74-&4S`J^Qk|^@VE()XnRANP@Ri?HGa@{`-L`lMiIam zH*Xd&)xmX0GGFM-u||+hLicqwhw^dyzvmYJ!}W~R_4%fK;D@_p1Y{GVUG-Ie)SvW* zO+IpkydZl+Ac}}Nm0(kx^T!B0Kgm}_ZiYk!j-VDx+tv1*zxO;_S&VE-FxcEG*z#1ZLLxdj`CGU-u@UQaPaS=pK1r_OTkYNMEY=GR5=jfyoQa-L;AMz=c zV^-`8kf=wZ)ukan^}Kd?Y}1EC#61;&jKqg)#7IocosI8$9h*Mv29qEt{a4f1#PoD4{FNE zH)ep2kQnO0D`u{4CLmvoL)U%GDf(>c1|~qzda&30uR9)GNG3iYd$!)!G*N8H)DAq2}ej4pWLB_|8UOyFIYG_ z?D9zoo7`wKJ{X>IHO~Amy6Nq$@e?N{)_;lI5#d69p$PTl7k!2u1~+)YMDxE^qj;fAh{6YP3gSYkBqT35BM3->4)G@%Z~rY<)66b@>rg@1vDfz<4JxP) z2i$WQVxDsn`4o8$0uGLY>p&HJbckv&1_yc=fIRAX>ruSN1p{~9Nftxx)f6XOf8la7 ztL^u)a5Bt0gL0aSJ7M2hYT226(%Z%MU|C-JK!)b_HZ;_dfpfR-#wOJP6W=Q6)6>@8 zu*qS~?9!;9{YPel7aZ5^@*WN#UGZ6pm#^!w@{6SVh}4ZXWs}GavEjh_v6H)4Nl)fN z&!5py4Z__e54XycN6_P+*v}x?7s;O$;WIm>?VDo8_qcxgHyMGWCMfKEfKdlJGJv72 zWH@tT{kA_ZmK$~6aAkT z^I)>i`#4@78o`aIfn-|OM?-0f#|Y!|KD>;7inH8TF+VT066)*zk`nPgoWUK#S$?ew&Bn|1i}R0r?NP%c?(Ufq=Mi=LDDoE72ny%y$0*=BVz-nN$isF zB$dEMX}gSGR^`}r;;itX%5Q-xkCC*E5$WcFUWUh7!0?RTVgtgYSDw(rSn3FY12l#y=q-MWszL!L|!(kEdl^RhkJXktO1Mi ztzqdtaz=3H)wgiaPZr6hLY{`o27{_^g!|R z<-u~#&kvV1HD!8}LDk~?<#*nKASdWJri$`d?U_@rEn*xJU0ZKFR7s^}oJm;5$ByXR zRo6~Vv2rJ#nONx%N>GAsxXV$fcfQ(OB+s~C;mJ@&Yuh)qi*xEqX6?Q6K?+IB`0ddx zbKgDM#@olgyVHZ(h4?vXd7*Y+i?V1|X}thX02i*jKwx%KubJ@#yR1CsHfxK{PpwQ$ z(ku0Tp_j#8qv)8*Y@aImT*OXPo)Q4v37N5iTO0nSrdQjKwz(TkD&1yQx`a$DmQF5K z23obvk&6mc<+N(&!e;ZnHBp0^f=t`?#<9JS&MkB`$|)Dcf6BE?DtOgJJ-lGOk1pN#t|)v{e`AB)0Dp}F zv>IceyxC8eTexO)8#3V67XC@sQ{%>2Us(NjD)@iDxjrbc=FQ?2z^tBy+|)z8{HfSN z#HMelaquENbAtjxu;U6cw4>e@^{h#<>${K*zgX#!)Aqp)!{HL$#|Qfa%Fn=x%@kaoF+?1XnLlPRidq5{zrHBFEc&*xZXcy zdY3j|vsOH$%y)SGhQgw$qoj-R?YPT?$&|Cyx+dZs;aK2ns2PA@~qos zRR5BX=%}{iEo!$s<8NQSLO5qH;r}m_zckff4X?ZRZnresI=&=;h=JtCJ&siW#nrRNf%J^GnUm-J)jwbD zT+Rp083ib3OWW4C2w@{@-(ZcPj)?)yM)pe`x{))nHC?s46gq&79)fTc+AS=)4x~2M z=w+Y6h`6We-i~DvxfKIzjz#aE*_ADRr0r2sg;JYS!?w3%+gR;p24jqF0mnMGau;ue z`G^i0Y{#`121_w$+XgIb8pQL5r8z;PtiJBzxXnu20GYb-E~K}X)qkU`#J^IyW!_-r$yGd}t5|9wRTMCSlbGTzZ#SErUZAMwX@Brw$v?V%H zFkmRObaKs!ZkVCM5&`A-C(pRomrB=RS|94&CO%k9wwDAMn(&P;zPmXxLjOSe>PFvrU?$TMWy&u5djXNW8x(pusfS59JsC%8 zKBcPT6pX-56Xo;ah-8m;!kD43J{PE{8{|8@@7=c(xUK=RG_EL4jn$Y)WZ4`^M*xIi z!~Dt*ZLZgUWd^=}e251qggQate7z_xFIsox^Pqxy64+8uCEDB&6a1qmWde?k7;aEy z$qltnGVk$X#j4`~$P+O#1(>`m@r^+fT)a= z!5!L~b(-*W>=OfA9RggP1TC(i;&|l;d?4Ki9k!hZQd(o*n6b-x%kKBZfd*TG&>pjgf|!<*5#rNn{e;P&jme^(<&*sTcKP zGIDX^dEuyepD|fofFFg4Tc;eAh<_e3a8lOq=pz7zSty)8mO@RN8`>h3QhA-?dVNBR zVJO_$rf~xF(dzab?T6k28jzXPJx8spD{b1-BqDDbaljEb=m~MX7B*|%uMcE`hTSu{ znx0=LlS?TN{a+h7Klw9yg#qVLkOqee+K83B?2jj7^&M`MB26CCeKbCSA&cNzF46m6 z5JlP~&}|aFi}>B=qRDe>HxM<>2y<{xJEz8yl}$-Dw?rh;ei!Jm8QHj8^|S0I0S1Jz zO|oBYZ?fJonYZ5trZ^1t8!4Q>Rt#=NQiw$ylsn1(2lYJmX_YL=3^)IJ4{eF$)OZ-H zjw_Bv*Ij6}FRg4Lrvc?PUH~1&x{uMK_a~U){-LADjbV9JPD_yVUvrU7$5oK{BQ|_r z8zsQOS)2kbx>n!TFuCE3kbC#xDS+5RK&(yYm~FMcsr*lY>*2s&v{IvW`R9z3^NN_F zehS}rH;JKsDARX7HVPUv1IHV>6Y5L857GiRT|ic7<4greVVWGZr!w}lT?H@{sOzWV z{(WT6(Sahn*L5+9n>J?)R%rr#EaH`d)}&MJAZejHP6=Ky%K^#)(ZG#!YUGL=?vo|) z!x0!mWqZ;Se

$_b1MQ#oK&8wv4o^mu??56b8>%$@ql9-Bz=MLgvAaR+fH4W2ei# zZ4^fH`7Z(-v3B~No_%{~cA#TAVH+r0467mc-ZAh-=|P$dpQN1%HwZe%`sB-{bfpyb zrzUT(#T{XGyY5BtfU;W_!VO$K{43?n)g*xrSoo1TPFm%bkIZZC;}f*{6QayPp9pCy+ZfkH^-f#d&BW$bJPApWtL(Y!BNJSqcQ-NY>8 zHY)N0DC@jH(ZA1zaHfPGmqLEj`TY?Ga03%Rw-cQ@O7X}1(mTJSo~Z%Lj!ti@y(uRw zdZ%l%n20lBJ_e9i`h>EcqHL`TKsiR=WYTI45Enp~$7o*5hFFoa)h^7DC+z^=Gl)eO zs^I0P<=nSGmeg7ZWT^m7Ni)H87=Ab;ogk28*(qf6(h7RlKckOaUjAJ{@JUYs=hc0Ws)O~g$gcTjRx(6 zEQE?y`_uY8zmGF%uR~S)v&N&-rbei144O~%#(*3^;~=Rn8ew{_=7Am~0p@hFlo`zE zohRJL8z@Y_KblUnlGw)`CCINDJ8HV-5mV9F*j}5R-e}Pz&117#&1BH$sEd5_gT3Zr zD5!21v*AsY>uSfoU zo+heotvxQ6$860@Z{v{8sV_@iOKV(f8x4(L7gkF70DcK{L#Gp)g(#Q%*^a1yAHC7V z@H%w|EThxmdM%CZ;|09+FSob2x5pcmUCoPaYh=*)GI|-6)Xqt-_Qrj4!975mA=Sv7 zLYtHxi4}kPyb|BZLJSNd_Bl8Nn6U{L#to0_?J^2|xKz0m>@YmnxH8fRm}xJ{3EiOP zdOoMMIsG}$6Yzo)^k3F(!9VN&md6b93Uok0O#2_9qyJAT=)$9XETNMlcsO1s_%5Z1SDlctq(vuO-%-U*O-r3(Zu zWi2E|0 zu=$evGM(LKh!qz-x<;dqE~lpv$!d;xlQ~&Aq_C|F^!diW{}cCr=a=yRw`Hv8r41XM X*zoFU=40@e0kDgj`Wl7ktM~pF-WYUY literal 0 HcmV?d00001 From 4161e8d73aa49f61c4b63aa251a4bc4ec66b0571 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Fri, 9 Dec 2022 16:54:11 +0000 Subject: [PATCH 033/114] Move doc files to correct directory --- QueryManagement.md => docs/QueryManagement.md | 0 .../graphics/torq-qlm_architecture.PNG | Bin 2 files changed, 0 insertions(+), 0 deletions(-) rename QueryManagement.md => docs/QueryManagement.md (100%) rename torq-qlm_architecture.PNG => docs/graphics/torq-qlm_architecture.PNG (100%) diff --git a/QueryManagement.md b/docs/QueryManagement.md similarity index 100% rename from QueryManagement.md rename to docs/QueryManagement.md diff --git a/torq-qlm_architecture.PNG b/docs/graphics/torq-qlm_architecture.PNG similarity index 100% rename from torq-qlm_architecture.PNG rename to docs/graphics/torq-qlm_architecture.PNG From ca9c5a709edadf1c0f08ea39208fcee00abfd9d1 Mon Sep 17 00:00:00 2001 From: acreehay Date: Mon, 12 Dec 2022 14:27:02 +0000 Subject: [PATCH 034/114] added password files for new procs --- config/passwords/qtp.txt | 1 + config/passwords/queryfeed.txt | 1 + config/passwords/querygateway.txt | 1 + config/passwords/queryhdb.txt | 1 + config/passwords/queryrdb.txt | 1 + 5 files changed, 5 insertions(+) create mode 100644 config/passwords/qtp.txt create mode 100644 config/passwords/queryfeed.txt create mode 100644 config/passwords/querygateway.txt create mode 100644 config/passwords/queryhdb.txt create mode 100644 config/passwords/queryrdb.txt diff --git a/config/passwords/qtp.txt b/config/passwords/qtp.txt new file mode 100644 index 000000000..5171c10f4 --- /dev/null +++ b/config/passwords/qtp.txt @@ -0,0 +1 @@ +querytp:pass diff --git a/config/passwords/queryfeed.txt b/config/passwords/queryfeed.txt new file mode 100644 index 000000000..02e2087f4 --- /dev/null +++ b/config/passwords/queryfeed.txt @@ -0,0 +1 @@ +queryfeed:pass diff --git a/config/passwords/querygateway.txt b/config/passwords/querygateway.txt new file mode 100644 index 000000000..0d1247cd5 --- /dev/null +++ b/config/passwords/querygateway.txt @@ -0,0 +1 @@ +querygateway:pass diff --git a/config/passwords/queryhdb.txt b/config/passwords/queryhdb.txt new file mode 100644 index 000000000..9e21bd1eb --- /dev/null +++ b/config/passwords/queryhdb.txt @@ -0,0 +1 @@ +queryhdb:pass diff --git a/config/passwords/queryrdb.txt b/config/passwords/queryrdb.txt new file mode 100644 index 000000000..ea3ecb8ef --- /dev/null +++ b/config/passwords/queryrdb.txt @@ -0,0 +1 @@ +queryrdb:pass From aca69cd67b94e9fc46f589b46340d6abf25cc697 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Tue, 13 Dec 2022 14:08:33 +0000 Subject: [PATCH 035/114] Change process type qtp to querytp --- code/handlers/trackqueries.q | 48 ----- code/processes/queryfeed.q | 4 +- code/querygateway/daqrest.q | 42 +++++ code/querygateway/dataaccess.q | 211 ++++++++++++++++++++++ code/querygateway/gatewaylib.q | 94 ++++++++++ code/querygateway/kxdash.q | 61 +++++++ code/querygateway/querygatewayfuncs.q | 0 code/{qtp => querytp}/pubsub.q | 0 code/{qtp => querytp}/sctp.q | 0 code/{qtp => querytp}/stplog.q | 0 code/{qtp => querytp}/stpmeta.q | 0 config/filealerterprocessed/.d | Bin 0 -> 34 bytes config/filealerterprocessed/filename | Bin 0 -> 4112 bytes config/filealerterprocessed/filename# | Bin 0 -> 4246 bytes config/filealerterprocessed/filesize | Bin 0 -> 32 bytes config/filealerterprocessed/md5hash | Bin 0 -> 4112 bytes config/filealerterprocessed/md5hash# | Bin 0 -> 4164 bytes config/passwords/{qtp.txt => querytp.txt} | 0 config/settings/queryfeed.q | 6 +- config/settings/queryrdb.q | 4 +- config/settings/{qtp.q => querytp.q} | 0 21 files changed, 417 insertions(+), 53 deletions(-) delete mode 100644 code/handlers/trackqueries.q create mode 100644 code/querygateway/daqrest.q create mode 100644 code/querygateway/dataaccess.q create mode 100644 code/querygateway/gatewaylib.q create mode 100644 code/querygateway/kxdash.q create mode 100644 code/querygateway/querygatewayfuncs.q rename code/{qtp => querytp}/pubsub.q (100%) rename code/{qtp => querytp}/sctp.q (100%) rename code/{qtp => querytp}/stplog.q (100%) rename code/{qtp => querytp}/stpmeta.q (100%) create mode 100644 config/filealerterprocessed/.d create mode 100644 config/filealerterprocessed/filename create mode 100644 config/filealerterprocessed/filename# create mode 100644 config/filealerterprocessed/filesize create mode 100644 config/filealerterprocessed/md5hash create mode 100644 config/filealerterprocessed/md5hash# rename config/passwords/{qtp.txt => querytp.txt} (100%) rename config/settings/{qtp.q => querytp.q} (100%) diff --git a/code/handlers/trackqueries.q b/code/handlers/trackqueries.q deleted file mode 100644 index 4ec848a03..000000000 --- a/code/handlers/trackqueries.q +++ /dev/null @@ -1,48 +0,0 @@ -\d .queries - -// flags and variables -enabled:@[value;`enabled;1b] -ignorequery:@[value;`ignorequery;1b] -ignorequerylist:@[value;`ignorequerylist;(`upd;)] -ignoreuser:@[value;`ignoreuser;1b] -ignoreuserlist:@[value;`ignoreuserlist;(`discovery;`segmentedtickerplant;`rdb;`hdb;`sort;`gateway;`monitor;`housekeeping;`reporter;`filealerter;`feed;`segmentedchainedtickerplant;`sortworker;`metrics;`iexfeed;`dqc;`dqcdb;`dqe;`dqedb;`tailer_seg1;`tailer_seg2;`tr_seg1;`tr_seg2;`tailsort;`;.z.u)] -timerval:@[value;`settimer;0D00:00:10] -threshold:@[value;`threshold;50] - -// table schema -queries:@[value;`queries;([]starttime:`timestamp$();endtime:`timestamp$();runtime:`long$();user:`symbol$();ip:`int$();prochost:`symbol$();procname:`symbol$();proctype:`symbol$();query:();success:`boolean$())] - -// upsert successful query -logquery:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;1b); result} -// upsert failed query -logqueryerror:{[endp;result;arg;startp] `.queries.queries upsert (startp;endp;`long$.001*endp-startp;.z.u;.z.a;.z.h;.proc.procname;.proc.proctype;arg;0b); 'result} - -p1:{.queries.logquery[.proc.cp[];@[x;y;.queries.logqueryerror[.proc.cp[];;y;startp]];y;startp:.proc.cp[]]} -// for ignoring users/procs and/or specific queries -p2:{if[ignoreuser; if[any .z.u~/:ignoreuserlist; :x@y]]; if[ignorequery; if[any y~/:ignorequerylist; :x@y]]; p1[x;y]} - -if[enabled; .z.pg:(p2).z.pg; .z.ps:(p2).z.ps;]; - -// functionality to save down tables to disk -savedown:{[tabname;pt] - dir:`$":",getenv `KDBHDB; - pth:` sv .Q.par[dir;pt;tabname],`; - numrows: count .queries.queries; - .lg.o[`save;"saving ",(string numrows)," rows of " (string tabname)," data to partition ", string pt] - err:{[e].lg.e[`savedata;"Failed to save client query data to disk : ",e];'e}; - if[numrows > 0;.[upsert;(pth;.Q.en[dir;.save.manipulate[tabname;.queries.queries]]);err];]; - .queries.queries:0#.queries.queries; - }; - -rowcheck:{[tabname;pt] - countvalue:count .queries.queries; - if[countvalue > .queries.threshold; savedown[tabname;pt]]; - }; - -// timers set up to periodically save down to disk -if[enabled; - settimers:{ - .timer.repeat[.proc.cp[];0Wp;0D00:00:10;(`.queries.rowcheck;`clientqueries;.z.d);"save client query data to disk"]; - .timer.repeat[.proc.cp[];0Wp;.queries.timerval;(`.queries.savedown;`clientqueries;.z.d);"save client query data to disk"]; - }; - .queries.settimers[];]; diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index 6db8f2de6..18024a38f 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -38,8 +38,8 @@ flushreload:{ {h(".u.upd";`usage;value flip select from readlog[raze string (getenv `KDBLOG),"/usage_",(raze x),"_",.z.d,".log"])} each string each procnames; }; -.servers.startupdepcycles[`qtp;10;0W]; -h:.servers.gethandlebytype[`qtp;`any]; +.servers.startupdepcycles[`querytp;10;0W]; +h:.servers.gethandlebytype[`querytp;`any]; if[reloadenabled;.timer.once[.proc.cp[]+0D00:00:10.000;(`flushreload;`);"Flush reload"]]; .timer.repeat[.proc.cp[];0Wp;0D00:00:00.200;(`queryfeed;`);"Publish Query Feed"]; diff --git a/code/querygateway/daqrest.q b/code/querygateway/daqrest.q new file mode 100644 index 000000000..ef64ab08b --- /dev/null +++ b/code/querygateway/daqrest.q @@ -0,0 +1,42 @@ +\d .dataaccess + +// .gw.formatresponse:{[status;sync;result]$[not[status]and sync;'result;result]}}; + +//Gets the json and converts to input dict before executing .dataaccess.getdata on the input +qrest:{ + // Set the response type + .gw.formatresponse:{[status;sync;result] $[sync and not status; 'result; `status`result!(status;result)]}; + // Run the function + :getdata jsontodict x}; +// Converts json payload to .dataaaccess input dictionary +jsontodict:{ + // convert the input to a dictionary + dict:.j.k x; + k:key dict; + // Change the Type of `tabname`instruments`grouping to chars + dict:@[dict;`tablename`instruments`grouping`columns inter k;{`$x}]; + // Change the Type of `start/end time to timestamps (altering T -> D and - -> . if applicable) + dict:@[dict;`starttime`endtime inter k;{x:ssr[x;"T";"D"];x:ssr[x;"-";"."];value x}]; + // retrieve aggregations + if[`aggregations in k;dict[`aggregations]:value dict[`aggregations]]; + // Convert timebar + if[`timebar in k;dict[`timebar]:@[value dict[`timebar];1+til 2;{:`$x}]]; + // Convert the filters key + if [`filters in k;dict[`filters]:filterskey dict`filters]; + //output + :dict}; + +quotefinder:{y[2#where y>x]} + +filterskey:{[filtersstrings] + likelist:ss[filtersstrings;"like"]; + if[0=count likelist;value filtersstrings]; + // Get the location of all the backticks + apostlist:ss[filtersstrings;"'"]; + // Get the location of all the likes + swaplist:raze {y[2#where y>x]}[;apostlist] each likelist; + // Swap the ' to " + filtersstrings:@[filtersstrings;swaplist;:;"\""]; + // Convert the string to a dict + :value filtersstrings + }; diff --git a/code/querygateway/dataaccess.q b/code/querygateway/dataaccess.q new file mode 100644 index 000000000..3d7054acf --- /dev/null +++ b/code/querygateway/dataaccess.q @@ -0,0 +1,211 @@ +\d .dataaccess + +forceservers:0b; + +// dictionary containing aggregate functions needed to calculate map-reducable +// values over multiple processes +aggadjust:(!). flip( + (`avg; {flip(`sum`count;2#x)}); + (`cor; {flip(`wsum`count`sum`sum`sumsq`sumsq;@[x;(enlist(0;1);0;0;1;0;1)])}); + (`count; `); + (`cov; {flip(`wsum`count`sum`sum;@[x;(enlist(0;1);0;0;1)])}); + (`dev; {flip(`sumsq`count`sum;3#x)}); + (`distinct;`); + (`first; `); + (`last; `); + (`max; `); + (`min; `); + (`prd; `); + (`sum; `); + (`var; {flip(`sumsq`count`sum;3#x)}); + (`wavg; {flip(`wsum`sum;(enlist(x 0;x 1);x 0))}); + (`wsum; {enlist(`wsum;enlist(x 0;x 1))})); + +// function to make symbols strings with an upper case first letter +camel:{$[11h~type x;@[;0;upper]each string x;@[string x;0;upper]]}; +// function that creates aggregation where X(X1,X2)=X(X(X1),X(X2)) where X is +// the aggregation and X1 and X2 are non overlapping subsets of a list +absagg:{enlist[`$x,y]!enlist(value x;`$x,y)}; + +// functions to calculate avg, cov and var in mapaggregate dictionary +avgf:{(%;(sum;`$"sum",x);scx y)}; +covf:{(-;(%;swsum[x;y];scx x);(*;avgf[x;x];avgf[y;x]))}; +varf:{(-;(%;(sum;`$"sumsq",y);scx x);(xexp;avgf[y;x];2))}; +// functions to sum counts and wsums in mapaggregate dictioanry +scx:{(sum;`$"count",x)}; +swsum:{(sum;`$"wsum",x,y)} + +// dictionary containing the functions needed to aggregate results together for +// map reducable aggregations +mapaggregate:(!). flip( + (`avg; {enlist[`$"avg",x]!enlist(%;(sum;`$"sum",x);scx x)}); + (`cor; {enlist[`$"cor",x,w]!enlist(%;covf[x;w];(*;(sqrt;varf[x;x]);(sqrt;varf[(x:x 0);w:x 1])))}); + (`count; {enlist[`$"count",x]!enlist scx x}); + (`cov; {enlist[`$"cov",x,w]!enlist covf[x:x 0;w:x 1]}); + (`dev; {enlist[`$"dev",x]!enlist(sqrt;varf[x;x])}); + (`first; {enlist[`$"first",x]!enlist(*:;`$"first",x)}); + (`last; {absagg["last";x]}); + (`max; {absagg["max";x]}); + (`min; {absagg["min";x]}); + (`prd; {absagg["prd";x]}); + (`sum; {absagg["sum";x]}); + (`var; {enlist[`$"var",x]!enlist varf[x;x]}); + (`wavg; {enlist[`$"wavg",x,w]!enlist(%;swsum[x:x 0;w:x 1];(sum;`$"sum",x))}); + (`wsum; {enlist[`$"wsum",x,w]!enlist swsum[x:x 0;w:x 1]})); + +// function to convert sorting +go:{if[`asc=x[0];:(xasc;x[1])];:(xdesc;x[1])}; + +// Full generality dataaccess function in the gateway +getdata:{[o] + // Input checking in the gateway + reqno:.requests.initlogger[o]; + o:@[.checkinputs.checkinputs;o;.requests.error[reqno]]; + // Get the Procs + if[not `procs in key o;o[`procs]:attributesrouting[o;partdict[o]]]; + // Get Default process behavior + default:`timeout`postback`sublist`getquery`queryoptimisation`postprocessing!(0Wn;();0W;0b;1b;{:x;}); + // Use upserting logic to determine behaviour + options:default,o; + if[`ordering in key o;options[`ordering]: go each options`ordering]; + o:adjustqueries[o;partdict o]; + options[`mapreduce]:0b; + gr:$[`grouping in key options;options`grouping;`]; + if[`aggregations in key options; + if[all key[options`aggregations]in key aggadjust; + options[`mapreduce]:not`date in gr]]; + // Execute the queries + if[options`getquery; + $[.gw.call .z.w; + :.gw.syncexec[(`.dataaccess.buildquery;o);options[`procs]]; + :.gw.asyncexec[(`.dataaccess.buildquery;o);options[`procs]]]]; + :$[.gw.call .z.w; + // if sync + .gw.syncexecjt[(`getdata;o);options[`procs];autojoin[options];options[`timeout]]; + // if async + .gw.asyncexecjpt[(`getdata;o);options[`procs];autojoin[options];options[`postback];options[`timeout]]]; + }; + + +// join results together if from multiple processes +autojoin:{[options] + // if there is only one proc queried output the table + if[1=count options`procs;:first]; + // if there is no need for map reducable adjustment, return razed results + :$[not options`mapreduce;razeresults[options;];mapreduceres[options;]]; + }; + +// raze results and call process res to apply postprocessing and sublist +razeresults:{[options;res] + res:raze res; + processres[options;res] + }; + +//apply sublist and post processing to joined results +processres:{[options;res] + res:(options`postprocessing)res; + :$[(options`sublist)<>0W;(options`sublist) sublist res;res]; + }; + +// function to correctly reduce two tables to one +mapreduceres:{[options;res] + // raze the result sets together + res:$[all 99h=type each res; + (){x,0!y}/res; + (),/res]; + aggs:options`aggregations; + aggs:flip(key[aggs]where count each value aggs;raze aggs); + // distinct may be present as only agg, so apply distinct again + if[all`distinct=first each aggs;:?[res;();1b;()]]; + // collecting the appropriate grouping argument for map-reduce aggs + gr:$[all`grouping`timebar in key options; + a!a:options[`timebar;2],options`grouping; + `grouping in key options; + a!a:(),options`grouping; + `timebar in key options; + a!a:(),options[`timebar;2]; + 0b]; + // select aggs by gr from res + res:?[res;();gr;raze{mapaggregate[x 0;camel x 1]}'[aggs]]; + //apply sublist and postprocesing to map reduced results + processres[options;res] + }; + + +// Dynamic routing finds all processes with relevant data +attributesrouting:{[options;procdict] + // Get the tablename and timespan + timespan:`date$options[`starttime`endtime]; + // See if any of the provided partitions are with the requested ones + procdict:{[x;timespan] (all x within timespan) or any timespan within x}[;timespan] each procdict; + // Only return appropriate dates + types:(key procdict) where value procdict; + // If the dates are out of scope of processes then error + if[0=count types; + '`$"gateway error - no info found for that table name and time range. Either table does not exist; attributes are incorect in .gw.servers on gateway, or the date range is outside the ones present" + ]; + :types; + }; + +// Generates a dictionary of `tablename!mindate;maxdate +partdict:{[input] + tabname:input[`tablename]; + // Remove duplicate servertypes from the gw.servers + servers:select from .gw.servers where i=(first;i)fby servertype; + // extract the procs which have the table defined + servers:select from servers where {[x;tabname]tabname in @[x;`tables]}[;tabname] each attributes; + // Create a dictionary of the attributes against servertypes + procdict:(exec servertype from servers)!(exec attributes from servers)@'(key each exec attributes from servers)[;0]; + // If the response is a dictionary index into the tablename + procdict:@[procdict;key procdict;{[x;tabname]if[99h=type x;:x[tabname]];:x}[;tabname]]; + // returns the dictionary as min date/ max date + procdict:asc @[procdict;key procdict;{:(min x; max x)}]; + // prevents overlap if more than one process contains a specified date + if[1count p:options`procs;:options]; + // get the date casting where relevant + st:$[a:-14h~tp:type start:options`starttime;start;`date$start]; + et:$[a;options`endtime;`date$options`endtime]; + // get the dates that are required by each process + dates:group key[part]where each{within[y;]each value x}[part]'[l:st+til 1+et-st]; + dates:l{(min x;max x)}'[dates]; + // if start/end time not a date, then adjust dates parameter for the + // correct types + if[not a; + // converts dates dictionary to timestamps/datetimes + dates:$[-15h~tp;{"z"$x};::]{(0D+x 0;x[1]+1D-1)}'[dates]; + // convert first and last timestamp to start and end time + dates:@[dates;f;:;(start;dates[f:first key dates;1])]; + dates:@[dates;l;:;(dates[l:last key dates;0];options`endtime)]]; + // adjust map reducable aggregations to get correct components + if[(1type att; + // its a list of servertypes e.g. `rdb`hdb + // check if user attributes are a symbol list + if[not 11h=abs type att; + '" Servertype should be given as either a dictionary(type 99h) or a symbol list (11h)" + ]; + servertype:distinct att,(); + //list of active servers + activeservers:exec distinct servertype from .gw.servers where active; + //list of all servers + allservers:exec distinct servertype from .gw.servers; + activeserversmsg:". Available servers include: ",", " sv string activeservers; + //check if a null argument is passed + if[any null att;'"A null server cannot be passed as an argument",activeserversmsg]; + //if any requested servers are missing then: + //if requested server does not exist, return error with list of available servers + //if requested server exists but is currently inactive, return error with list of available servers + if[count servertype except activeservers; + '"the following ",$[max not servertype in allservers; + "are not valid servers: ",", " sv string servertype except allservers; + "requested servers are currently inactive: ",", " sv string servertype except activeservers + ],activeserversmsg; + ]; + :(exec serverid by servertype from .gw.servers where active)[servertype]; + ]; + + serverids:$[`servertype in key att; + raze getserveridstype[delete servertype from att] each (),att`servertype; + getserveridstype[att;`all]]; + + if[all 0=count each serverids;'"no servers match requested attributes"]; + :serverids; + } + +getserveridstype:{[att;typ] + // default values + besteffort:1b; + attype:`cross; + + servers:$[typ=`all; + exec serverid!attributes from .gw.servers where active; + exec serverid!attributes from .gw.servers where active,servertype=typ]; + + if[`besteffort in key att; + if[-1h=type att`besteffort;besteffort:att`besteffort]; + att:delete besteffort from att; + ]; + if[`attributetype in key att; + if[-11h=type att`attributetype;attype:att`attributetype]; + att:delete attributetype from att; + ]; + + res:$[attype=`independent;getserversindependent[att;servers;besteffort]; + getserverscross[att;servers;besteffort]]; + + serverids:first value flip $[99h=type res; key res; res]; + if[all 0=count each serverids;'"no servers match ",string[typ]," requested attributes"]; + :serverids; + } + +/- build a cross product from a nested dictionary +buildcross:{(cross/){flip (enlist y)#x}[x] each key x} + +/- given a dictionary of requirements and a list of attribute dictionaries +/- work out which servers we need to hit to satisfy each requirement +/- we want to satisfy the cross product of requirements - so each attribute has to be available with each other attribute +/- e.g. each symbol has to be availble within each specified date +getserverscross:{[req;att;besteffort] + + if[0=count req; :([]serverid:enlist key att)]; + + s:getserversinitial[req;att]; + + /- build the cross product of requirements + reqcross:buildcross[req]; + + /- calculate the cross product of data contributed by each source + /- and drop it from the list of stuff that is required + util:flip `remaining`found!flip ({[x;y;z] (y[0] except found; y[0] inter found:$[0=count y[0];y[0];buildcross x@'where each z])}[req]\)[(reqcross;());value s]; + + /- check if everything is done + if[(count last util`remaining) and not besteffort; + '"getserverscross: cannot satisfy query as the cross product of all attributes can't be matched"]; + /- remove any rows which don't add value + s:1!(0!s) w:where not 0=count each util`found; + /- return the parameters which should be queried for + (key s)!distinct each' flip each util[w]`found + } diff --git a/code/querygateway/kxdash.q b/code/querygateway/kxdash.q new file mode 100644 index 000000000..d978aa509 --- /dev/null +++ b/code/querygateway/kxdash.q @@ -0,0 +1,61 @@ +\d .kxdash +enabled:@[value;`enabled;{0b}]; + +// use this to store the additional params that the kx dashboards seem to send in +dashparams:`o`w`r`limit!(0;0i;0i;0W) + +// function to be called from the dashboards +dashexec:{[q;s;j] + .gw.asyncexecjpt[(dashremote;q;dashparams);(),s;dashjoin[j];();0Wn] + } + +// execute the request +// return a dict of status and result, along with the params +// add a flag to the start of the list to stop dictionaries collapsing +// to tables in the join function +dashremote:{[q;dashparams] + (`kxdash;dashparams,`status`result!@[{(1b;value x)};q;{(0b;x)}]) + } + +// join function used for dashboard results +dashjoin:{[joinfunc;r] + $[min r[;1;`status]; + (`.dash.rcv_msg;r[0;1;`w];r[0;1;`o];r[0;1;`r];r[0;1;`limit] sublist joinfunc r[;1;`result]); + (`.dash.snd_err;r[0;1;`w];r[0;1;`r];r[0;1;`result])] + } + +dashps:{ + // check the query coming in meets the format + $[@[{`f`w`r`x`u~first 1_ value first x};x;0b]; + // pull out the values we need to return to the dashboards + [dashparams::`o`w`r`limit!(last value x 1;x 2;x 3;x[4;0]); + // execute the query part, which must look something like + // .kxdash.dashexec["select from t";`rdb`hdb;raze] + value x[4;1]; + ]; + // + value x] + } + + +// need this to handle queries that only hit one backend process +// reformat those responses to look the same +formatresponse:{[status;sync;result] + if[`kxdash~first result; + res:last result; + :$[res`status; + (`.dash.rcv_msg;res`w;res`o;res`r;res`result); + (`.dash.snd_err;res`w;res`r;res`result)]]; + $[not[status]and sync;'result;result]} + + +init:{ + // KX dashboards are expecting getFunctions to be defined on the process + .api.getFunctions:@[value;`.api.getFunctions;{{:()}}]; + // Reset format response + .gw.formatresponse:formatresponse; + // incorporate dashps into the .z.ps definition + .z.ps:{x@y;.kxdash.dashps y}@[value;`.z.ps;{{value x}}]; + }; + +if[enabled;init[]]; diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q new file mode 100644 index 000000000..e69de29bb diff --git a/code/qtp/pubsub.q b/code/querytp/pubsub.q similarity index 100% rename from code/qtp/pubsub.q rename to code/querytp/pubsub.q diff --git a/code/qtp/sctp.q b/code/querytp/sctp.q similarity index 100% rename from code/qtp/sctp.q rename to code/querytp/sctp.q diff --git a/code/qtp/stplog.q b/code/querytp/stplog.q similarity index 100% rename from code/qtp/stplog.q rename to code/querytp/stplog.q diff --git a/code/qtp/stpmeta.q b/code/querytp/stpmeta.q similarity index 100% rename from code/qtp/stpmeta.q rename to code/querytp/stpmeta.q diff --git a/config/filealerterprocessed/.d b/config/filealerterprocessed/.d new file mode 100644 index 0000000000000000000000000000000000000000..5a956ca530ccfb60a664072a5514a91171976205 GIT binary patch literal 34 mcmey*$j!jaz`&4}nUk8An48Lwn_`-gSeyYCD9)@(WdH!Aa|%@e literal 0 HcmV?d00001 diff --git a/config/filealerterprocessed/filename b/config/filealerterprocessed/filename new file mode 100644 index 0000000000000000000000000000000000000000..6142ea359bfaf803a9537bc804b2b087b89882dc GIT binary patch literal 4112 zcmeIuu?YYm6hlGdB<>^bKg|b0D_4+9m-1bWW&{WjAV7cs0RjXF5FkK+z-I!Lb&C4i Hm3e} literal 0 HcmV?d00001 diff --git a/config/filealerterprocessed/filename# b/config/filealerterprocessed/filename# new file mode 100644 index 0000000000000000000000000000000000000000..cc7706d3a03534a4f216ead6647ee86d16347f79 GIT binary patch literal 4246 zcmeIuF$#k~6a-L<9K;hi=nVokDFrXEtUo01bwBPZc!V5!LA0|^B{SVL1M_lOFchEw z1t>rP3Q&Lo6rcbFC_n)U+)N-BkISRq3!g9woL^LBg4eH1sas^;T3PXqhCxijhCa;Q Z(+uj8#02xsBBC2jrLKJ9UDgg)$04=cr{r~^~ literal 0 HcmV?d00001 diff --git a/config/filealerterprocessed/md5hash b/config/filealerterprocessed/md5hash new file mode 100644 index 0000000000000000000000000000000000000000..219066472de216702173ef38a1cf61f29afe7dc3 GIT binary patch literal 4112 zcmeIuu?YYm6hlGdB;r2qKg|b0D_4+9m-1bWW&{WjAV7cs0RjXF5FkK+z-I!Lb&C4i Hm3eWvB`VWs@Ddq$r_P=7;)sW|o4Rlg_a@p33vqq{26q$v literal 0 HcmV?d00001 diff --git a/config/passwords/qtp.txt b/config/passwords/querytp.txt similarity index 100% rename from config/passwords/qtp.txt rename to config/passwords/querytp.txt diff --git a/config/settings/queryfeed.q b/config/settings/queryfeed.q index 849919545..1fa5c552b 100644 --- a/config/settings/queryfeed.q +++ b/config/settings/queryfeed.q @@ -1,7 +1,11 @@ \d .servers enabled:1b -CONNECTIONS:`qtp; // Feedhandler connects to the query-tickerplant +CONNECTIONS:`querytp; // Feedhandler connects to the query-tickerplant HOPENTIMEOUT:30000 \d . subprocs:"S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.csv"; // List of procs for query-tickerplant to subscribe to +reloadenabled:1b + +\d .usage +enabled:0b diff --git a/config/settings/queryrdb.q b/config/settings/queryrdb.q index b4bcb789c..d0eb23cc4 100644 --- a/config/settings/queryrdb.q +++ b/config/settings/queryrdb.q @@ -3,7 +3,7 @@ ignorelist:`heartbeat`logmsg`.usage.usage //list of tables to ignore when saving to disk hdbtypes:`queryhdb //list of hdb types to look for and call in hdb reload hdbnames:() //list of hdb names to search for and call in hdb reload -tickerplanttypes:`qtp //list of tickerplant types to try and make a connection to +tickerplanttypes:`querytp //list of tickerplant types to try and make a connection to gatewaytypes:`querygateway //list of gateway types to try and make a connection to checktpperiod:0D00:00:05 //how often to check for tickerplant connection onlyclearsaved:0b //if true, eod writedown will only clear tables which have been successfully saved to disk @@ -34,4 +34,4 @@ loadprocesscode:1b // whether to load the process specific code defined at ${KD // Server connection details \d .servers CONNECTIONS:`queryhdb`querygateway // list of connections to make at start up -STARTUP:1b // create connections \ No newline at end of file +STARTUP:1b // create connections diff --git a/config/settings/qtp.q b/config/settings/querytp.q similarity index 100% rename from config/settings/qtp.q rename to config/settings/querytp.q From 1661c80c2e6f33e7b717193987c96cd64d8ac946 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Fri, 16 Dec 2022 15:28:38 +0000 Subject: [PATCH 036/114] Change runtime column to timespan --- code/processes/queryfeed.q | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index 18024a38f..e20b99a52 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -3,9 +3,9 @@ // add connections to all procs for query tracking to be enabled .servers.CONNECTIONS:.servers.CONNECTIONS,exec distinct proctype from (" SS ";enlist csv) 0: hsym `$getenv `TORQPROCESSES where procname in subprocs; -us:@[value;`us;([]querytime:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$();procname:`symbol$;status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())]; +us:@[value;`us;([]querytime:`timestamp$();id:`long$();runtime:`timespan$();zcmd:`symbol$();proctype:`symbol$();procname:`symbol$;status:`char$();ip:`int$();user:`symbol$();handle:`int$();cmd:();mem:();sz:`long$();error:())]; -upd:{[t;x] if [t in `.usage.usage; `us insert x]}; +upd:{[t;x] x[2]:`timespan$x[2]; if [t in `.usage.usage; `us insert x]}; .servers.startup[]; start_sub:{[subprocs] @@ -35,7 +35,7 @@ queryfeed:{ flushreload:{ .lg.o[`flushreload1;"fr1"]; procnames:exec distinct procname from .servers.SERVERS where proctype in subprocs; - {h(".u.upd";`usage;value flip select from readlog[raze string (getenv `KDBLOG),"/usage_",(raze x),"_",.z.d,".log"])} each string each procnames; + {h(".u.upd";`usage;value flip update timer:`timespan$1000*timer from readlog[raze string (getenv `KDBLOG),"/usage_",(raze x),"_",.z.d,".log"])} each string each procnames; }; .servers.startupdepcycles[`querytp;10;0W]; From b22278cdee3f204f2e0a533b882f3e2a5f6f9166 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Wed, 21 Dec 2022 17:47:57 +0000 Subject: [PATCH 037/114] Add column breakdown to docs --- docs/QueryManagement.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/QueryManagement.md b/docs/QueryManagement.md index 69dcc1369..0d40831cf 100644 --- a/docs/QueryManagement.md +++ b/docs/QueryManagement.md @@ -54,3 +54,34 @@ Query Gateway ------------- The Query Gateway process subscribes to our specific Query RDB and HDB processes to load balance queries and allow for access to query information involving both historical and real time data. The query gateway process also contains a number of analytics functions to return specific aggregations from the usage table (i.e how many queries were executed for each process on a specific day). + +Table Contents +============== + +The Query usage table contains the follow columns: + +- time: The time that the query was received or when it finished executing + +- runtime: The time taken for the query to be executed + +- zcmd: The .z function that was invoked + +- proctype: The process type of the process being queried + +- procname: The name of the process being queried + +- status: Denotes whether the entry is before the query is executed (b), that the query has been successfully executed (c) or if the query execution was in error (e) + +- ip: The IP address of the process being queried + +- user: The user who is executing the query + +- handle: The handle over which the query is passed + +- cmd: The query and its arguments as a string + +- mem: Memory information connected to the query execution + +- sz: The byte size of the result from the query being executed + +- error: An error string From b335f5119f892a3e21e3afae9739e4af899a6154 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Thu, 22 Dec 2022 16:10:56 +0000 Subject: [PATCH 038/114] Add setup information to docs --- docs/QueryManagement.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/QueryManagement.md b/docs/QueryManagement.md index 0d40831cf..ad4982a10 100644 --- a/docs/QueryManagement.md +++ b/docs/QueryManagement.md @@ -85,3 +85,20 @@ The Query usage table contains the follow columns: - sz: The byte size of the result from the query being executed - error: An error string + +Setup +===== + +The TorQ Query Management logging tool can be added on top of any existing TorQ setup or the full TorQ application including Query logging can be integrated with data sources and feeds. + +As mentioned above, there are 5 processes that need to be initialised: Query Tickerplant, Query RDB, Query Feed, Query HDB and a Query Gateway. An example of how these would be added to a process csv file is displayed below: + + localhost,{KDBBASEPORT}+24,querytp,querytp1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/querydatabase.q -tplogdir ${KDBTPLOG} + localhost,{KDBBASEPORT}+25,queryrdb,queryrdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBCODE}/processes/rdb.q,1, + localhost,{KDBBASEPORT}+26,queryfeed,queryfeed1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBCODE}/processes/queryfeed.q,1, + localhost,{KDBBASEPORT}+27,queryhdb,queryhdb1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBQUERYHDB},1, + localhost,{KDBBASEPORT}+28,querygateway,querygateway1,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,4000,${KDBCODE}/processes/querygateway.q,1, + +Each of these processes have their own unique process types and process files (except the Query RDB which uses the rdb.q file). Functionality for these 5 processes can be refined within each process config file. + +The Query Management logging tool is fully customisable so users can select specific processes that they want queries to be monitored for. To activate this functionality for a process, add the process name to the querytrack.csv file which is located in the TorQ/config/ directory. This csv file is loaded into the query feed process and used to activate Query Management Logging for each of the specified processes. From fcb92c1c7166816131c9121d27ee3842ebd8d33c Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Thu, 22 Dec 2022 16:32:12 +0000 Subject: [PATCH 039/114] Add query gateway functions script --- code/querygateway/querygatewayfuncs.q | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index e69de29bb..beac891d8 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -0,0 +1,66 @@ +//Number of queries on a given process by hour (or any given bucket) +testfunc:{.gw.syncexec[({[x;y]select query_suc:count i by procname from usage where time within (x;y)};x;y);`queryrdb]} +//Number of queries ran by a specific user +//Number of distinct users querying a process +//Return list of queries that run over a given run time +//Return the date range queried on a process + + +/Number of queries on a given process by hour (or any given bucket) +//d:(`proc`bucket`sd`ed)!(`rdb1:60:.z.d-3;.z.d) +ProcSucErr:{[d] + .gw.syncexec[({[d] + $[.proc.proctype=`queryhdb; + select queries_suc:count where status=first string `c,queries_err:count where status=first string `e + by time.date,d[`bucket] xbar time.minute from usage + where date within (d[`sd];d[`ed]),procname=d[`proc]; + select queries_suc:count where status=first string `c,queries_err:count where status=first string `e + by time.date,d[`bucket] xbar time.minute from usage + where procname=d[`proc]]};d);`queryhdb`queryrdb]; + }; +//ProcSucErrFF:{[`proc;`bucket;sd;ed] +// .gw.syncexec[({[`proc;`bucket;sd;ed] +// $[.proc.proctype=`queryhdb; +// +// ?[`usage;((within;`date;(enlist;`sd;`ed));(=;`procname;`proc)); +// `date`minute!(`time.date;(k){x*y div x:$[16h=abs[@x];"j"$x;x]};`bucket;`time.minute); +// `queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]; +// +// ?[`usage;enlist((=;`procname;`proc)); +// `date`minute!(`time.date;(k){x*y div x:$[16h=abs[@x];"j"$x;x]};`bucket;`time.minute); +// `queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]]};`proc;`bucket;sd;ed);`queryhdb`queryrdb]; +// }; +//Number of queries ran by a specific user +//d:(`user`sd`ed)!(`acreehay;.z.d-3;.z.d) +QueryByUser:{[d] + .gw.syncexec[({[d] + $[.proc.proctype=`queryhdb; + select queries_suc:count where status=first string `c,queries_err:count where status=first string `e + by time.date,procname from usage + where date within (d[`sd];d[`ed]),u=d[`user]; + select queries_suc:count where status=first string `c,queries_err:count where status=first string `e + by time.date,procname from usage + where u=d[`user]]};d);`queryhdb`queryrdb]; + }; +//Number of distinct users querying a process +//d:(`proc`bucket`sd`ed)!(`rdb1;60;.z.d-3;.z.d) +DistinctUsers:{[d] + .gw.syncexec[({[d] + $[.proc.proctype=`queryhdb; + select unique_users:count distinct user, users:distinct user + by time.date,d[`bucket] xbar time.minute from usage + where date within (d[`sd];d[`ed]),procname=d[`proc]; + select unique_users:count distinct user, users:distinct user + by time.date,d[`bucket] xbar time.minute from usage + where procname=d[`proc]]};d);`queryhdb`queryrdb]; + }; +//Return list of queries that run over a given run time +//10000;.z.d-3;.z.d +TimerLimit:{[lim;sd;ed] + .gw.syncexec[({[lim;sd;ed] + $[.proc.proctype=`queryhdb; + select from usage + where date within (sd;ed),runtime>lim; + select from usage + where runtime>lim]};lim;sd;ed);`queryhdb`queryrdb]; + }; From bd3775e50db2a8a7b9d49a453134e7713408dd81 Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Thu, 22 Dec 2022 17:41:36 +0000 Subject: [PATCH 040/114] Typo fix in Query gateway functions script --- code/querygateway/querygatewayfuncs.q | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index beac891d8..6a27e72c9 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -7,7 +7,7 @@ testfunc:{.gw.syncexec[({[x;y]select query_suc:count i by procname from usage wh /Number of queries on a given process by hour (or any given bucket) -//d:(`proc`bucket`sd`ed)!(`rdb1:60:.z.d-3;.z.d) +//d:(`proc`bucket`sd`ed)!(`rdb1;60:.z.d-3;.z.d) ProcSucErr:{[d] .gw.syncexec[({[d] $[.proc.proctype=`queryhdb; @@ -37,10 +37,10 @@ QueryByUser:{[d] $[.proc.proctype=`queryhdb; select queries_suc:count where status=first string `c,queries_err:count where status=first string `e by time.date,procname from usage - where date within (d[`sd];d[`ed]),u=d[`user]; + where date within (d[`sd];d[`ed]),user=d[`user]; select queries_suc:count where status=first string `c,queries_err:count where status=first string `e by time.date,procname from usage - where u=d[`user]]};d);`queryhdb`queryrdb]; + where user=d[`user]]};d);`queryhdb`queryrdb]; }; //Number of distinct users querying a process //d:(`proc`bucket`sd`ed)!(`rdb1;60;.z.d-3;.z.d) From 762e0240c208b8a10a12b083fb86d0e38ecabc3e Mon Sep 17 00:00:00 2001 From: AlexCullenAquaQ Date: Fri, 23 Dec 2022 17:10:38 +0000 Subject: [PATCH 041/114] Add query gateway functions in functional form --- code/querygateway/querygatewayfuncs.q | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 6a27e72c9..5e55c8903 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -42,6 +42,17 @@ QueryByUser:{[d] by time.date,procname from usage where user=d[`user]]};d);`queryhdb`queryrdb]; }; + +//QuerByUser function in functional form +//Same arguments as above just not in dictionary form +QueryByUserFF:{[username;sd;ed] + .gw.syncexec[({[username;sd;ed] + $[.proc.proctype=`queryhdb; + ?[`usage;((within;`date;(enlist;`sd;`ed));(=;`user;`username));`date`procname!`time.date`procname;`queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]; + ?[`usage;enlist((=;`user;`username));`date`procname!`time.date`procname;`queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]]};username;sd;ed);`queryhdb`queryrdb]; + }; + + //Number of distinct users querying a process //d:(`proc`bucket`sd`ed)!(`rdb1;60;.z.d-3;.z.d) DistinctUsers:{[d] @@ -56,7 +67,7 @@ DistinctUsers:{[d] }; //Return list of queries that run over a given run time //10000;.z.d-3;.z.d -TimerLimit:{[lim;sd;ed] +RuntimeLimit:{[lim;sd;ed] .gw.syncexec[({[lim;sd;ed] $[.proc.proctype=`queryhdb; select from usage @@ -64,3 +75,11 @@ TimerLimit:{[lim;sd;ed] select from usage where runtime>lim]};lim;sd;ed);`queryhdb`queryrdb]; }; + +//RuntimeLimit function in functional form +RuntimeLimitFF:{[lim;sd;ed] + .gw.syncexec[({[lim;sd;ed] + $[.proc.proctype=`queryhdb; + ?[`usage;((within;`date;(enlist;`sd;`ed));(>;`runtime;`lim));0b;()]; + ?[`usage;enlist((>;`runtime;`lim));0b;()]]};lim;sd;ed);`queryhdb`queryrdb]; + }; From 3dd3908a946a326053207a21ad9c12a843ea432f Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 20 Jan 2023 11:38:47 +0000 Subject: [PATCH 042/114] Add missing proc name change --- config/settings/queryrdb1.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings/queryrdb1.q b/config/settings/queryrdb1.q index 32bc00312..a526493a0 100644 --- a/config/settings/queryrdb1.q +++ b/config/settings/queryrdb1.q @@ -4,7 +4,7 @@ ignorelist:`heartbeat`logmsg //list of tables to ignore when saving to disk hdbtypes:`hdb //list of hdb types to look for and call in hdb reload hdbnames:() //list of hdb names to search for and call in hdb reload -tickerplanttypes:`qtp //list of tickerplant types to try and make a connection to +tickerplanttypes:`querytp //list of tickerplant types to try and make a connection to gatewaytypes:`gateway //list of gateway types to try and make a connection to checktpperiod:0D00:00:05 //how often to check for tickerplant connection onlyclearsaved:0b //if true, eod writedown will only clear tables which have been successfully saved to disk From e65c2f584e5c15146c34229c2fe43028ba3f962b Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 20 Jan 2023 16:29:55 +0000 Subject: [PATCH 043/114] Add GetDateRange fn --- code/querygateway/querygatewayfuncs.q | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 5e55c8903..a2e44eb7d 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -83,3 +83,15 @@ RuntimeLimitFF:{[lim;sd;ed] ?[`usage;((within;`date;(enlist;`sd;`ed));(>;`runtime;`lim));0b;()]; ?[`usage;enlist((>;`runtime;`lim));0b;()]]};lim;sd;ed);`queryhdb`queryrdb]; }; + +GetDateRange:{[query] + wherephrase:raze raze (parse query) 2; + dateindex:1 + wherephrase?`date; + // index into wherephrase to retrieve date + date:wherephrase dateindex; + + // if date is a range, drop the 'enlist' element + if[3=count date; date:1_date]; + + :date; + }; From 04d78a8dd82a237cfdb0b958a3898bf93ce2d71a Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 24 Jan 2023 10:41:33 +0000 Subject: [PATCH 044/114] Fix to handle .z args --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index a2e44eb7d..f4f314483 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -93,5 +93,5 @@ GetDateRange:{[query] // if date is a range, drop the 'enlist' element if[3=count date; date:1_date]; - :date; + :eval each date; }; From d20f0268e69413f00f2dd67efc4b5010451ea110 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 13:20:28 +0000 Subject: [PATCH 045/114] Merge query count fns from torq_qlm_test --- code/querygateway/querygatewayfuncs.q | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index f4f314483..9267e16c0 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -95,3 +95,22 @@ GetDateRange:{[query] :eval each date; }; + +QueryCountsRealtime:{ + query:"select queries:count u by u from usage where u in `angus`michael`stephen"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:handle query; + :res; + }; + +QueryCountsHistorical:{[date] + $[.z.d<=date; query:(); // log error + 1=count date; query:"select queries:count u by u from usage where date=", string date, ", u in `angus`michael`stephen"; + 2=count date; query:"select queries:count u by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; + // log error + query:()] + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:handle raze query; + :res; + }; + From 1ec4d1b84c1dad1035930f63b82e817483d3d08b Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 13:35:48 +0000 Subject: [PATCH 046/114] Delete config/filealerterprocessed/ directory --- config/filealerterprocessed/.d | Bin 34 -> 0 bytes config/filealerterprocessed/filename | Bin 4112 -> 0 bytes config/filealerterprocessed/filename# | Bin 4246 -> 0 bytes config/filealerterprocessed/filesize | Bin 32 -> 0 bytes config/filealerterprocessed/md5hash | Bin 4112 -> 0 bytes config/filealerterprocessed/md5hash# | Bin 4164 -> 0 bytes 6 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 config/filealerterprocessed/.d delete mode 100644 config/filealerterprocessed/filename delete mode 100644 config/filealerterprocessed/filename# delete mode 100644 config/filealerterprocessed/filesize delete mode 100644 config/filealerterprocessed/md5hash delete mode 100644 config/filealerterprocessed/md5hash# diff --git a/config/filealerterprocessed/.d b/config/filealerterprocessed/.d deleted file mode 100644 index 5a956ca530ccfb60a664072a5514a91171976205..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34 mcmey*$j!jaz`&4}nUk8An48Lwn_`-gSeyYCD9)@(WdH!Aa|%@e diff --git a/config/filealerterprocessed/filename b/config/filealerterprocessed/filename deleted file mode 100644 index 6142ea359bfaf803a9537bc804b2b087b89882dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4112 zcmeIuu?YYm6hlGdB<>^bKg|b0D_4+9m-1bWW&{WjAV7cs0RjXF5FkK+z-I!Lb&C4i Hm3e} diff --git a/config/filealerterprocessed/filename# b/config/filealerterprocessed/filename# deleted file mode 100644 index cc7706d3a03534a4f216ead6647ee86d16347f79..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4246 zcmeIuF$#k~6a-L<9K;hi=nVokDFrXEtUo01bwBPZc!V5!LA0|^B{SVL1M_lOFchEw z1t>rP3Q&Lo6rcbFC_n)U+)N-BkISRq3!g9woL^LBg4eH1sas^;T3PXqhCxijhCa;Q Z(+uj8#02xsBBC2jrLKJ9UDgg)$04=cr{r~^~ diff --git a/config/filealerterprocessed/md5hash b/config/filealerterprocessed/md5hash deleted file mode 100644 index 219066472de216702173ef38a1cf61f29afe7dc3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4112 zcmeIuu?YYm6hlGdB;r2qKg|b0D_4+9m-1bWW&{WjAV7cs0RjXF5FkK+z-I!Lb&C4i Hm3eWvB`VWs@Ddq$r_P=7;)sW|o4Rlg_a@p33vqq{26q$v From d464604c04a2aa7f2be888efa18fdcfa8ee84004 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 14:38:42 +0000 Subject: [PATCH 047/114] Add total query count fn --- code/querygateway/querygatewayfuncs.q | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 9267e16c0..4c2d1a421 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -97,13 +97,20 @@ GetDateRange:{[query] }; QueryCountsRealtime:{ + query:"select count i from usage where u in `angus`michael`stephen"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:handle query; + :res; + }; + +QueryUserCountsRealtime:{ query:"select queries:count u by u from usage where u in `angus`michael`stephen"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; }; -QueryCountsHistorical:{[date] +QueryUserCountsHistorical:{[date] $[.z.d<=date; query:(); // log error 1=count date; query:"select queries:count u by u from usage where date=", string date, ", u in `angus`michael`stephen"; 2=count date; query:"select queries:count u by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; From 965c28d2afeeb43dee3861c19005cb94298ed6c2 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 16:00:58 +0000 Subject: [PATCH 048/114] Ignore config/filealerterprocessed/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e43b0f988..267d73112 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .DS_Store +config/filealerterprocessed/ From 4eab35e497c50ab5baa99ec2783930a2d59ad385 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 16:02:47 +0000 Subject: [PATCH 049/114] Add peak usage fn --- code/querygateway/querygatewayfuncs.q | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 4c2d1a421..b495bc0c0 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -121,3 +121,9 @@ QueryUserCountsHistorical:{[date] :res; }; +PeakUsage:{ + query:"0!select count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:handle query; + :res; + }; From bc8a9a83997138b64b1e575947add8478cfdf820 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 16:14:34 +0000 Subject: [PATCH 050/114] Rename peak usage col names --- code/querygateway/querygatewayfuncs.q | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index b495bc0c0..d70471ffe 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -122,7 +122,8 @@ QueryUserCountsHistorical:{[date] }; PeakUsage:{ - query:"0!select count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; + query:"0!select queries:count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; + query:`time xcol tab; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; From b12d1ab6fe5e7c203d0f98346adecb737ee00906 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 16:17:26 +0000 Subject: [PATCH 051/114] Fix typo --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index d70471ffe..a5f657fb6 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -123,7 +123,7 @@ QueryUserCountsHistorical:{[date] PeakUsage:{ query:"0!select queries:count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; - query:`time xcol tab; + query:`time xcol query; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; From b75991d7c9ee7b2bf9676243bec735b5fd2def38 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 26 Jan 2023 16:22:00 +0000 Subject: [PATCH 052/114] Fix peak usage query --- code/querygateway/querygatewayfuncs.q | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index a5f657fb6..4adc86d51 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -122,8 +122,7 @@ QueryUserCountsHistorical:{[date] }; PeakUsage:{ - query:"0!select queries:count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; - query:`time xcol query; + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; From 651d1d48ad3a6805a578f2456a4a4d950b8db8ee Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 27 Jan 2023 12:27:20 +0000 Subject: [PATCH 053/114] Add peak usage function --- code/querygateway/querygatewayfuncs.q | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 4adc86d51..4251ba5a0 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -124,6 +124,24 @@ QueryUserCountsHistorical:{[date] PeakUsage:{ query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:handle query; + res::handle query; + + time::select distinct time from res; + querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[`angus`michael`stephen]; + querycountsn:{x xcol y}'[`angus`michael`stephen; querycounts]; + querycountsnk:{`time xkey ![x;();0b;(enlist `time)!enlist (each; raze; `time)]}'[querycountsn]; + peakusage:0!(lj/)(querycountsnk); + + :update time:.z.d + time from peakusage; + }; + +QueryCountsHistorical:{[date] + $[.z.d<=date; query:(); // log error + 1=count date; query:"select queries:count u by u from usage where date=", string date, ", u in `angus`michael`stephen"; + 2=count date; query:"select queries:count u by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; + // log error + query:()] + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:handle raze query; :res; }; From c076962aab8c15faa87a83f2e5aa96ceb3cefcd1 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 27 Jan 2023 12:34:47 +0000 Subject: [PATCH 054/114] Fix formatting --- code/querygateway/querygatewayfuncs.q | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 4251ba5a0..965b2b950 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -104,16 +104,27 @@ QueryCountsRealtime:{ }; QueryUserCountsRealtime:{ - query:"select queries:count u by u from usage where u in `angus`michael`stephen"; + query:"select queries:count i by u from usage where u in `angus`michael`stephen"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; }; +QueryCountsHistorical:{[date] + $[.z.d<=date; query:(); // log error + 1=count date; query:"select queries:count i from usage where date=", string date, ", u in `angus`michael`stephen"; + 2=count date; query:"select queries:count i from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; + // log error + query:()] + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:handle raze query; + :res; + }; + QueryUserCountsHistorical:{[date] $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count u by u from usage where date=", string date, ", u in `angus`michael`stephen"; - 2=count date; query:"select queries:count u by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; + 1=count date; query:"select queries:count i by u from usage where date=", string date, ", u in `angus`michael`stephen"; + 2=count date; query:"select queries:count i by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; @@ -134,14 +145,3 @@ PeakUsage:{ :update time:.z.d + time from peakusage; }; - -QueryCountsHistorical:{[date] - $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count u by u from usage where date=", string date, ", u in `angus`michael`stephen"; - 2=count date; query:"select queries:count u by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; - // log error - query:()] - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - res:handle raze query; - :res; - }; From 60abde634128a8031d8435f34b02587240ac58b8 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 27 Jan 2023 14:37:31 +0000 Subject: [PATCH 055/114] Fix type error in update stmt --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 965b2b950..5303fa654 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -140,7 +140,7 @@ PeakUsage:{ time::select distinct time from res; querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[`angus`michael`stephen]; querycountsn:{x xcol y}'[`angus`michael`stephen; querycounts]; - querycountsnk:{`time xkey ![x;();0b;(enlist `time)!enlist (each; raze; `time)]}'[querycountsn]; + querycountsnk:{`time xkey ![x;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}'[querycountsn]; peakusage:0!(lj/)(querycountsnk); :update time:.z.d + time from peakusage; From f9ef13b0d08c5209c96f742ec848b27e81b9ef51 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 30 Jan 2023 16:36:43 +0000 Subject: [PATCH 056/114] Add longest running query fn --- code/querygateway/querygatewayfuncs.q | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 5303fa654..008dd066e 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -145,3 +145,22 @@ PeakUsage:{ :update time:.z.d + time from peakusage; }; + +// if there are multiple queries this just grabs the "first" of them +// assuming this is acceptable given the likelihood of two offensively long queries +// having exactly the same runtime +LongestRunning:{ + query:"select runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=max runtime"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:handle query; + + cmd:raze value flip select cmd from res; + r:.z.d + raze value flip select runtime from res; + u:raze raze value flip select u from res; + + f:`$2_first ";" vs raze cmd; + q:first next "\"" vs raze cmd; + p:`$-1_last "`" vs raze cmd; + + :([] runtime:r; user:u; func:enlist f; query:enlist q; proc:enlist p); + }; From e41b5d3a46c3ff929e3d173fbf9225523ec435c5 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Wed, 1 Feb 2023 18:33:38 +0000 Subject: [PATCH 057/114] Add longest running heatmap fn --- code/querygateway/querygatewayfuncs.q | 42 +++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 008dd066e..c4bc575a1 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -94,7 +94,32 @@ GetDateRange:{[query] if[3=count date; date:1_date]; :eval each date; - }; + }; + +// return the function, query and process from the cmd col of usage tak +ParseCmd:{[res] + cmd:raze value flip select cmd from res; + + f:`$2_first ";" vs raze cmd; + q:first next "\"" vs raze cmd; + p:`$-1_last "`" vs raze cmd; + + :(`func`query`proc)!enlist each (f; q; p); + }; + +ParseCmdImproved:{[res] + cmd:raze value flip select cmd from res; + remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; + reslist:select cmd from update cmd:";" vs ' cmd from res; + + // grab function, query and process and parse out any + // brackets or slashes in strings + f:`$1_'first each first value flip reslist; + q:1_'-1_'first each next each first value flip reslist; + p:`$-1_'first each next each next each first value flip reslist; + + :remainder,' ([] f; q; p); + }; QueryCountsRealtime:{ query:"select count i from usage where u in `angus`michael`stephen"; @@ -154,13 +179,18 @@ LongestRunning:{ handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; - cmd:raze value flip select cmd from res; r:.z.d + raze value flip select runtime from res; u:raze raze value flip select u from res; - f:`$2_first ";" vs raze cmd; - q:first next "\"" vs raze cmd; - p:`$-1_last "`" vs raze cmd; + cmdparsed:ParseCmd res; + + :([] runtime:r; user:u; func:cmdparsed `func; query:cmdparsed `query; proc:cmdparsed `proc); + }; + +LongestRunningHeatMap:{ + query:"update minute:.z.d + minute from 0!select max runtime, first cmd by 10 xbar time.minute from usage where u in `angus`michael`stephen"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:handle query; - :([] runtime:r; user:u; func:enlist f; query:enlist q; proc:enlist p); + :cmdparsed:ParseCmdImproved res; }; From 8ba256a2041f64e5af5e91af9a3e6f0a6a5f522d Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Wed, 1 Feb 2023 18:35:05 +0000 Subject: [PATCH 058/114] Remove assignment --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index c4bc575a1..62a5f9878 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -192,5 +192,5 @@ LongestRunningHeatMap:{ handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; - :cmdparsed:ParseCmdImproved res; + :ParseCmdImproved res; }; From 00092ab951ff03a2dcae4c9b51b2f25438c19bd8 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Feb 2023 16:04:32 +0000 Subject: [PATCH 059/114] Update select stmt with Hamza's fix --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 62a5f9878..a0c62cec7 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -188,7 +188,7 @@ LongestRunning:{ }; LongestRunningHeatMap:{ - query:"update minute:.z.d + minute from 0!select max runtime, first cmd by 10 xbar time.minute from usage where u in `angus`michael`stephen"; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; From be45878e1349e585ad6a9c1a1dd17c4ab0ddebc4 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Feb 2023 16:14:10 +0000 Subject: [PATCH 060/114] Remove old cmd parser --- code/querygateway/querygatewayfuncs.q | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index a0c62cec7..987e3b5c8 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -96,19 +96,8 @@ GetDateRange:{[query] :eval each date; }; -// return the function, query and process from the cmd col of usage tak ParseCmd:{[res] cmd:raze value flip select cmd from res; - - f:`$2_first ";" vs raze cmd; - q:first next "\"" vs raze cmd; - p:`$-1_last "`" vs raze cmd; - - :(`func`query`proc)!enlist each (f; q; p); - }; - -ParseCmdImproved:{[res] - cmd:raze value flip select cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; reslist:select cmd from update cmd:";" vs ' cmd from res; @@ -182,9 +171,7 @@ LongestRunning:{ r:.z.d + raze value flip select runtime from res; u:raze raze value flip select u from res; - cmdparsed:ParseCmd res; - - :([] runtime:r; user:u; func:cmdparsed `func; query:cmdparsed `query; proc:cmdparsed `proc); + :ParseCmd res; }; LongestRunningHeatMap:{ @@ -192,5 +179,5 @@ LongestRunningHeatMap:{ handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; - :ParseCmdImproved res; + :ParseCmd res; }; From 63742c031fbce4f4c597089023d4783eb398af07 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Feb 2023 16:18:10 +0000 Subject: [PATCH 061/114] Remove redundant comment --- code/querygateway/querygatewayfuncs.q | 3 --- 1 file changed, 3 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 987e3b5c8..045d90b82 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -160,9 +160,6 @@ PeakUsage:{ :update time:.z.d + time from peakusage; }; -// if there are multiple queries this just grabs the "first" of them -// assuming this is acceptable given the likelihood of two offensively long queries -// having exactly the same runtime LongestRunning:{ query:"select runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; From e539a536af16cc1e7c4ce45a078d79907234dc40 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Feb 2023 16:19:46 +0000 Subject: [PATCH 062/114] Remove redundant selects --- code/querygateway/querygatewayfuncs.q | 3 --- 1 file changed, 3 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 045d90b82..b34a72d65 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -165,9 +165,6 @@ LongestRunning:{ handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; - r:.z.d + raze value flip select runtime from res; - u:raze raze value flip select u from res; - :ParseCmd res; }; From bcd28e0bc31891076a98c585432f11d397013f2e Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Feb 2023 16:22:45 +0000 Subject: [PATCH 063/114] Add column names --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index b34a72d65..2b3ecb486 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -107,7 +107,7 @@ ParseCmd:{[res] q:1_'-1_'first each next each first value flip reslist; p:`$-1_'first each next each next each first value flip reslist; - :remainder,' ([] f; q; p); + :remainder,' ([] func:f; query:q; proc:p); }; QueryCountsRealtime:{ From 779244e5b1b26329c6d53280661bf35880162a64 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Feb 2023 17:42:35 +0000 Subject: [PATCH 064/114] Add time to longest running query fn --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 2b3ecb486..be06ff2a9 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -161,7 +161,7 @@ PeakUsage:{ }; LongestRunning:{ - query:"select runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=max runtime"; + query:"select time, runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; From 6fdbcbe9042257330ccbcca0d082bcbf2bea417f Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 16:15:18 +0000 Subject: [PATCH 065/114] Add ignore clients list --- code/handlers/logusage.q | 1 + config/settings/default.q | 1 + 2 files changed, 2 insertions(+) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index ec8b3f6a4..f5eb62da3 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -21,6 +21,7 @@ logtodisk:@[value;`logtodisk;1b] // whether to log to disk or not logtomemory:@[value;`logtomemory;1b] // write query logs to memory ignore:@[value;`ignore;1b] // check the ignore list for functions to ignore ignorelist:@[value;`ignorelist;(`upd;"upd")] // the list of functions to ignore +ignoreclients:@[value;`ignoreclient;`admin] // clients to ignore for usage monitoring flushinterval:@[value;`flushinterval;0D00:30:00] // default value for how often to flush the in-memory logs flushtime:@[value;`flushtime;0D03] // default value for how long to persist the in-memory logs suppressalias:@[value;`suppressalias;0b] // whether to suppress the log file alias creation diff --git a/config/settings/default.q b/config/settings/default.q index d5127a3c5..bb856bdd6 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -20,6 +20,7 @@ logtodisk:1b // whether to log to disk or not logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls +ignoreclients:`admin // clients to ignore for usage monitoring flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing suppressalias:0b // whether to suppress the log file alias creation From cf4d2f495e5f74cb95bc4999264562c43260c1f2 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 16:17:19 +0000 Subject: [PATCH 066/114] Add GetClients fn --- code/querygateway/querygatewayfuncs.q | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index be06ff2a9..4fba6ff60 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -96,6 +96,10 @@ GetDateRange:{[query] :eval each date; }; +GetClients:{ + :value flip select distinct u from .clients.clients where not u in .usage.ignoreclients; + }; + ParseCmd:{[res] cmd:raze value flip select cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; From 8f255d0446269fdd285f2df1df3575e88e7866e9 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 17:19:29 +0000 Subject: [PATCH 067/114] Add dynamic client tracking --- code/querygateway/querygatewayfuncs.q | 39 ++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 4fba6ff60..0253b1321 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -97,7 +97,9 @@ GetDateRange:{[query] }; GetClients:{ - :value flip select distinct u from .clients.clients where not u in .usage.ignoreclients; + clients:first value flip select distinct u from .clients.clients where not u in .usage.ignoreclients; + if[1=count clients; :first clients]; + :clients; }; ParseCmd:{[res] @@ -115,49 +117,60 @@ ParseCmd:{[res] }; QueryCountsRealtime:{ - query:"select count i from usage where u in `angus`michael`stephen"; + clients:GetClients[]; + query:"select count i from usage where u in ", (.Q.s1 clients); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; }; QueryUserCountsRealtime:{ - query:"select queries:count i by u from usage where u in `angus`michael`stephen"; + clients:GetClients[]; + query:"select queries:count i by u from usage where u in ", (.Q.s1 clients); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; }; QueryCountsHistorical:{[date] + clients:GetClients[]; + $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i from usage where date=", string date, ", u in `angus`michael`stephen"; - 2=count date; query:"select queries:count i from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; + 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 clients); + 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 clients); // log error query:()] + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:handle raze query; + :res; }; QueryUserCountsHistorical:{[date] + clients:GetClients; + $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i by u from usage where date=", string date, ", u in `angus`michael`stephen"; - 2=count date; query:"select queries:count i by u from usage where date within (", string first date, ";", string last date, "), u in `angus`michael`stephen"; + 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 clients); + 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 clients); // log error query:()] + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:handle raze query; + :res; }; PeakUsage:{ - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in `angus`michael`stephen"; + clients:GetClients[]; + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 clients); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res::handle query; time::select distinct time from res; - querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[`angus`michael`stephen]; - querycountsn:{x xcol y}'[`angus`michael`stephen; querycounts]; + querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[clients]; + querycountsn:{x xcol y}'[clients; querycounts]; querycountsnk:{`time xkey ![x;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}'[querycountsn]; peakusage:0!(lj/)(querycountsnk); @@ -165,7 +178,8 @@ PeakUsage:{ }; LongestRunning:{ - query:"select time, runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=max runtime"; + clients:GetClients[]; + query:"select time, runtime, u, cmd from usage where u in ", (.Q.s1 clients), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; @@ -173,7 +187,8 @@ LongestRunning:{ }; LongestRunningHeatMap:{ - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in `angus`michael`stephen, runtime=(max; runtime) fby 10 xbar time.minute"; + clients:GetClients[]; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (Q.s1 clients), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; From 2578530e7c2893bc156e89b253fa523e02dc529b Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 17:25:06 +0000 Subject: [PATCH 068/114] Fix typo --- code/querygateway/querygatewayfuncs.q | 2 +- config/settings/default.q | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 0253b1321..6b3896729 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -188,7 +188,7 @@ LongestRunning:{ LongestRunningHeatMap:{ clients:GetClients[]; - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (Q.s1 clients), ", runtime=(max; runtime) fby 10 xbar time.minute"; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 clients), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; diff --git a/config/settings/default.q b/config/settings/default.q index bb856bdd6..8258954b6 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -20,7 +20,7 @@ logtodisk:1b // whether to log to disk or not logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls -ignoreclients:`admin // clients to ignore for usage monitoring +ignoreclients:`admin // clients to ignore for usage monitoring flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing suppressalias:0b // whether to suppress the log file alias creation From ba1b2e33ab2ac5376f05c914840bf41f06f34fe7 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 17:39:49 +0000 Subject: [PATCH 069/114] Update name and value of ignoreclients --- code/handlers/logusage.q | 2 +- code/querygateway/querygatewayfuncs.q | 44 +++++++++++++-------------- config/settings/default.q | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index f5eb62da3..c70402277 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -21,7 +21,7 @@ logtodisk:@[value;`logtodisk;1b] // whether to log to disk or not logtomemory:@[value;`logtomemory;1b] // write query logs to memory ignore:@[value;`ignore;1b] // check the ignore list for functions to ignore ignorelist:@[value;`ignorelist;(`upd;"upd")] // the list of functions to ignore -ignoreclients:@[value;`ignoreclient;`admin] // clients to ignore for usage monitoring +ignoreusers:@[value;`ignoreusers;`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed] // clients to ignore for usage monitoring flushinterval:@[value;`flushinterval;0D00:30:00] // default value for how often to flush the in-memory logs flushtime:@[value;`flushtime;0D03] // default value for how long to persist the in-memory logs suppressalias:@[value;`suppressalias;0b] // whether to suppress the log file alias creation diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 6b3896729..cd50fdfc8 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -96,10 +96,10 @@ GetDateRange:{[query] :eval each date; }; -GetClients:{ - clients:first value flip select distinct u from .clients.clients where not u in .usage.ignoreclients; - if[1=count clients; :first clients]; - :clients; +GetUsers:{ + users:first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers; + if[1=count users; :first users]; + :users; }; ParseCmd:{[res] @@ -117,27 +117,27 @@ ParseCmd:{[res] }; QueryCountsRealtime:{ - clients:GetClients[]; - query:"select count i from usage where u in ", (.Q.s1 clients); + users:GetUsers[]; + query:"select count i from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; }; QueryUserCountsRealtime:{ - clients:GetClients[]; - query:"select queries:count i by u from usage where u in ", (.Q.s1 clients); + users:GetUsers[]; + query:"select queries:count i by u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; :res; }; QueryCountsHistorical:{[date] - clients:GetClients[]; + users:GetUsers[]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 clients); - 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 clients); + 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); + 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 users); // log error query:()] @@ -148,11 +148,11 @@ QueryCountsHistorical:{[date] }; QueryUserCountsHistorical:{[date] - clients:GetClients; + users:GetUsers[]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 clients); - 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 clients); + 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); + 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 users); // log error query:()] @@ -163,14 +163,14 @@ QueryUserCountsHistorical:{[date] }; PeakUsage:{ - clients:GetClients[]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 clients); + users:GetUsers[]; + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res::handle query; time::select distinct time from res; - querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[clients]; - querycountsn:{x xcol y}'[clients; querycounts]; + querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[users]; + querycountsn:{x xcol y}'[users; querycounts]; querycountsnk:{`time xkey ![x;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}'[querycountsn]; peakusage:0!(lj/)(querycountsnk); @@ -178,8 +178,8 @@ PeakUsage:{ }; LongestRunning:{ - clients:GetClients[]; - query:"select time, runtime, u, cmd from usage where u in ", (.Q.s1 clients), ", runtime=max runtime"; + users:GetUsers[]; + query:"select time, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; @@ -187,8 +187,8 @@ LongestRunning:{ }; LongestRunningHeatMap:{ - clients:GetClients[]; - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 clients), ", runtime=(max; runtime) fby 10 xbar time.minute"; + users:GetUsers[]; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:handle query; diff --git a/config/settings/default.q b/config/settings/default.q index 8258954b6..691515ba4 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -20,7 +20,7 @@ logtodisk:1b // whether to log to disk or not logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls -ignoreclients:`admin // clients to ignore for usage monitoring +ignoreusers:`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed // clients to ignore for usage monitoring flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing suppressalias:0b // whether to suppress the log file alias creation From c790f20649c2faec51bf97dd52c02f1ee84335c1 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 17:47:15 +0000 Subject: [PATCH 070/114] Add gateway to querygateway's connections --- config/settings/querygateway.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/settings/querygateway.q b/config/settings/querygateway.q index 7274d5b09..1b77a1080 100644 --- a/config/settings/querygateway.q +++ b/config/settings/querygateway.q @@ -17,8 +17,8 @@ loadprocesscode:1b // whether to load the process specific code def // Server connection details \d .servers -CONNECTIONS:`queryrdb`queryhdb // list of connections to make at start up +CONNECTIONS:`queryrdb`queryhdb`gateway // list of connections to make at start up RETRY:0D00:01 // period on which to retry dead connections. If 0, no reconnection attempts \d .aqrest -loadexecute:0b // Whether to reset .aqrest.execute \ No newline at end of file +loadexecute:0b // Whether to reset .aqrest.execute From 0cd64861ee458d8386fbf8ed1688bfc729ca6c6c Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 17:51:53 +0000 Subject: [PATCH 071/114] Update GetUsers fn to use normal gateway --- code/querygateway/querygatewayfuncs.q | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index cd50fdfc8..8ebc68540 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -97,9 +97,11 @@ GetDateRange:{[query] }; GetUsers:{ - users:first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers; - if[1=count users; :first users]; - :users; + query:"first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`gateway; + res:handle query; + if[1=count res; :first res]; + :res; }; ParseCmd:{[res] From 6d4b84ea988d5afeb740d59966968b7f57c4ccaa Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Mon, 6 Feb 2023 17:57:12 +0000 Subject: [PATCH 072/114] Add querygateway to ignoreusers --- code/handlers/logusage.q | 2 +- config/settings/default.q | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index c70402277..7d784b6f8 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -21,7 +21,7 @@ logtodisk:@[value;`logtodisk;1b] // whether to log to disk or not logtomemory:@[value;`logtomemory;1b] // write query logs to memory ignore:@[value;`ignore;1b] // check the ignore list for functions to ignore ignorelist:@[value;`ignorelist;(`upd;"upd")] // the list of functions to ignore -ignoreusers:@[value;`ignoreusers;`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed] // clients to ignore for usage monitoring +ignoreusers:@[value;`ignoreusers;`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway] // clients to ignore for usage monitoring flushinterval:@[value;`flushinterval;0D00:30:00] // default value for how often to flush the in-memory logs flushtime:@[value;`flushtime;0D03] // default value for how long to persist the in-memory logs suppressalias:@[value;`suppressalias;0b] // whether to suppress the log file alias creation diff --git a/config/settings/default.q b/config/settings/default.q index 691515ba4..e1ebe3047 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -20,7 +20,7 @@ logtodisk:1b // whether to log to disk or not logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls -ignoreusers:`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed // clients to ignore for usage monitoring +ignoreusers:`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway // clients to ignore for usage monitoring flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing suppressalias:0b // whether to suppress the log file alias creation From 32b1f13e8f7493e2008ecd48a2027078009089dc Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Feb 2023 13:48:23 +0000 Subject: [PATCH 073/114] Update sync queries to deferred sync --- code/querygateway/querygatewayfuncs.q | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 8ebc68540..ec3472532 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -122,7 +122,7 @@ QueryCountsRealtime:{ users:GetUsers[]; query:"select count i from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:handle query; + (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; :res; }; @@ -130,7 +130,7 @@ QueryUserCountsRealtime:{ users:GetUsers[]; query:"select queries:count i by u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:handle query; + (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; :res; }; @@ -144,7 +144,7 @@ QueryCountsHistorical:{[date] query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - res:handle raze query; + (neg handle)(`.gw.asyncexec; raze query; `queryhdb); res:handle[]; :res; }; @@ -159,7 +159,7 @@ QueryUserCountsHistorical:{[date] query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - res:handle raze query; + (neg handle)(`.gw.asyncexec; raze query; `queryhdb); res:handle[]; :res; }; @@ -168,7 +168,7 @@ PeakUsage:{ users:GetUsers[]; query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res::handle query; + (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; time::select distinct time from res; querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[users]; @@ -183,7 +183,7 @@ LongestRunning:{ users:GetUsers[]; query:"select time, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:handle query; + (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; :ParseCmd res; }; @@ -192,7 +192,7 @@ LongestRunningHeatMap:{ users:GetUsers[]; query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:handle query; + (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; :ParseCmd res; }; From b16a4cfd01be016bf71947e38713966899db6436 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Feb 2023 15:35:50 +0000 Subject: [PATCH 074/114] Fix deferred sync queries --- code/querygateway/querygatewayfuncs.q | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index ec3472532..1703bf57b 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -122,7 +122,7 @@ QueryCountsRealtime:{ users:GetUsers[]; query:"select count i from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; + res:raze last .async.deferred[handle; query]; :res; }; @@ -130,7 +130,7 @@ QueryUserCountsRealtime:{ users:GetUsers[]; query:"select queries:count i by u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; + res:raze last .async.deferred[handle; query]; :res; }; @@ -144,7 +144,7 @@ QueryCountsHistorical:{[date] query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - (neg handle)(`.gw.asyncexec; raze query; `queryhdb); res:handle[]; + res:raze last .async.deferred[handle; raze query]; :res; }; @@ -159,7 +159,7 @@ QueryUserCountsHistorical:{[date] query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - (neg handle)(`.gw.asyncexec; raze query; `queryhdb); res:handle[]; + res:raze last .async.deferred[handle; raze query]; :res; }; @@ -168,7 +168,7 @@ PeakUsage:{ users:GetUsers[]; query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; + res:raze last .async.deferred[handle; query]; time::select distinct time from res; querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[users]; @@ -183,8 +183,7 @@ LongestRunning:{ users:GetUsers[]; query:"select time, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; - + res:raze last .async.deferred[handle; query]; :ParseCmd res; }; @@ -192,7 +191,7 @@ LongestRunningHeatMap:{ users:GetUsers[]; query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - (neg handle)(`.gw.asyncexec; query; `queryrdb); res:handle[]; + res:raze last .async.deferred[handle; query]; :ParseCmd res; }; From 44ead6c3db9fac31e27b0ccd8b96fae4c1ca6f07 Mon Sep 17 00:00:00 2001 From: acreehay Date: Wed, 8 Feb 2023 09:55:22 +0000 Subject: [PATCH 075/114] removed redundant config file --- config/settings/queryrdb1.q | 40 ------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 config/settings/queryrdb1.q diff --git a/config/settings/queryrdb1.q b/config/settings/queryrdb1.q deleted file mode 100644 index a526493a0..000000000 --- a/config/settings/queryrdb1.q +++ /dev/null @@ -1,40 +0,0 @@ -// Bespoke RDB config - -\d .rdb -ignorelist:`heartbeat`logmsg //list of tables to ignore when saving to disk -hdbtypes:`hdb //list of hdb types to look for and call in hdb reload -hdbnames:() //list of hdb names to search for and call in hdb reload -tickerplanttypes:`querytp //list of tickerplant types to try and make a connection to -gatewaytypes:`gateway //list of gateway types to try and make a connection to -checktpperiod:0D00:00:05 //how often to check for tickerplant connection -onlyclearsaved:0b //if true, eod writedown will only clear tables which have been successfully saved to disk -subscribeto:` //a list of tables to subscribe to, default (`) means all tables -subscribesyms:` //a list of syms to subscribe for, (`) means all syms -savetables:1b //if true tables will be saved at end of day, if false tables wil not be saved, only wiped -garbagecollect:1b //if true .Q.gc will be called after each writedown - tradeoff: latency vs memory usage -upd:insert //value of upd -hdbdir:`:hdb //the location of the hdb directory -replaylog:1b //replay the tickerplant log file -schema:1b //retrieve the schema from the tickerplant -tpconnsleepintv:10 //number of seconds between attempts to connect to the tp -gc:1b //if true .Q.gc will be called after each writedown - tradeoff: latency vs memory usage - -sortcsv:hsym first .proc.getconfigfile["sort.csv"] //location of csv file -reloadenabled:0b //if true, the RDB will not save when .u.end is called but - //will clear it's data using reload function (called by the WDB) -parvaluesrc:`log //where to source the rdb partition value, can be log (from tp log file name), - //tab (from the the first value in the time column of the table that is subscribed for) - //anything else will return a null date which is will be filled by pardefault -pardefault:.z.D //if the src defined in parvaluesrc returns null, use this default date instead -tpcheckcycles:0W //specify the number of times the process will check for an available tickerplant -subfiltered:0b //allows subscription filters to be loaded and applied in the rdb -connectonstart:1b //rdb connects to tickerplant as soon as it is started - -\d .proc -loadprocesscode:1b // whether to load the process specific code defined at ${KDBCODE}/{process type} - -// Server connection details -\d .servers -CONNECTIONS:`hdb // list of connections to make at start up -STARTUP:1b // create connections - From fa356c6ea536f09dff2db8280a0fefec2ac1b8e4 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Wed, 8 Feb 2023 12:42:01 +0000 Subject: [PATCH 076/114] Add date getter for grafana --- code/querygateway/querygatewayfuncs.q | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 1703bf57b..f403feb2b 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -96,6 +96,12 @@ GetDateRange:{[query] :eval each date; }; +GetHistoricalDates:{ + query:"value flip select distinct date from usage"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + :last .async.deferred[handle; query]; + }; + GetUsers:{ query:"first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`gateway; From 2cf385fdc0f60cbc9ab0cc3698372923ee7e9ff1 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Wed, 8 Feb 2023 13:51:42 +0000 Subject: [PATCH 077/114] Remove redundant fn --- code/querygateway/querygatewayfuncs.q | 6 ------ 1 file changed, 6 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index f403feb2b..1703bf57b 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -96,12 +96,6 @@ GetDateRange:{[query] :eval each date; }; -GetHistoricalDates:{ - query:"value flip select distinct date from usage"; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - :last .async.deferred[handle; query]; - }; - GetUsers:{ query:"first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`gateway; From f9195f298747c762cb1a16c2f71bb7a97f047cdd Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Wed, 8 Feb 2023 15:16:31 +0000 Subject: [PATCH 078/114] Add QueryErrorPercentage fn --- code/querygateway/querygatewayfuncs.q | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index f403feb2b..c6239774f 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -201,3 +201,11 @@ LongestRunningHeatMap:{ :ParseCmd res; }; + +QueryErrorPercentage:{ + users:GetUsers[]; + query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where u in ", (.Q.s1 users); + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:raze last .async.deferred[handle; query]; + :res; + }; From a0a7cff19bb1b3e504800c1400b23b8996c35dec Mon Sep 17 00:00:00 2001 From: acreehay Date: Thu, 9 Feb 2023 10:50:08 +0000 Subject: [PATCH 079/114] added new hdbname to queryrdb config for hdb reload --- config/settings/queryrdb.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings/queryrdb.q b/config/settings/queryrdb.q index d0eb23cc4..166e1dc3c 100644 --- a/config/settings/queryrdb.q +++ b/config/settings/queryrdb.q @@ -2,7 +2,7 @@ \d .rdb ignorelist:`heartbeat`logmsg`.usage.usage //list of tables to ignore when saving to disk hdbtypes:`queryhdb //list of hdb types to look for and call in hdb reload -hdbnames:() //list of hdb names to search for and call in hdb reload +hdbnames:`queryhdb1 //list of hdb names to search for and call in hdb reload tickerplanttypes:`querytp //list of tickerplant types to try and make a connection to gatewaytypes:`querygateway //list of gateway types to try and make a connection to checktpperiod:0D00:00:05 //how often to check for tickerplant connection From 79cd3a0342937c1765af4a40c64cc85b0ffa1108 Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Thu, 9 Feb 2023 15:55:30 +0000 Subject: [PATCH 080/114] Add LongQuery and NumberOfUsers fn --- code/querygateway/querygatewayfuncs.q | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 89d36ba3c..1c6ffc35c 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -196,6 +196,7 @@ LongestRunningHeatMap:{ :ParseCmd res; }; +//Return percentage of queries that were successful by user QueryErrorPercentage:{ users:GetUsers[]; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where u in ", (.Q.s1 users); @@ -203,3 +204,23 @@ QueryErrorPercentage:{ res:raze last .async.deferred[handle; query]; :res; }; + +//Return queries which take longer than given runtime input, t +//t in milliseconds 10^-3 +LongQuery:{[t] + users:GetUsers[]; + timens:t*1000000; //Convert to nano seconds + query:"select from usage where runtime>", (.Q.s1 timens), ", u in ", (.Q.s1 users); + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:raze last .async.deferred[handle; query]; + :res; + }; + +//Number of distinct users +NumberOfUsers:{ + users:GetUsers[]; + query:"select count distinct u from usage where u in ", (.Q.s1 users); + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:raze last .async.deferred[handle; query]; + :res; + }; From 58891062817a4f0259eec8dad60a527e1308a12f Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 10 Feb 2023 16:02:36 +0000 Subject: [PATCH 081/114] Fix lexical scope issue --- code/querygateway/querygatewayfuncs.q | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 1c6ffc35c..8b9c058bb 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -169,13 +169,19 @@ PeakUsage:{ query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - - time::select distinct time from res; - querycounts:{?[`res; enlist (=; `u; enlist x); 0b; (enlist `queries)!(enlist `queries)]}'[users]; + + time:select distinct time from res; + + getquerycounts:{[res; users] ?[res; enlist (=; `u; enlist users); 0b; (enlist `queries)!(enlist `queries)]}[res; ]; + querycounts:getquerycounts'[users]; + querycountsn:{x xcol y}'[users; querycounts]; - querycountsnk:{`time xkey ![x;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}'[querycountsn]; + + getquerycountsnk:{[time; querycountsn] `time xkey ![querycountsn;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}[time; ]; + querycountsnk:getquerycountsnk'[querycountsn]; + peakusage:0!(lj/)(querycountsnk); - + :update time:.z.d + time from peakusage; }; From 2f9442965381dba4178e47a4344668aa36d042cd Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Mon, 13 Feb 2023 14:54:08 +0000 Subject: [PATCH 082/114] Add QueryErrorPercentageHistorical fn --- code/querygateway/querygatewayfuncs.q | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 8b9c058bb..b0e126885 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -211,6 +211,19 @@ QueryErrorPercentage:{ :res; }; +QueryErrorPercentageHistorical:{[date] + users:GetUsers[]; + $[.z.d<=date; query:(); // log error + 1=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); + 2=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 users); + // log error + query:()] + query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where u in ", (.Q.s1 users); + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:raze last .async.deferred[handle; query]; + :res; + }; + //Return queries which take longer than given runtime input, t //t in milliseconds 10^-3 LongQuery:{[t] From ef15caa61d3bbb2835ad60cb148eff7fd209131f Mon Sep 17 00:00:00 2001 From: acreehay Date: Tue, 14 Feb 2023 05:54:00 +0000 Subject: [PATCH 083/114] added queryhdb reload function --- code/queryhdb/queryhdbstandard.q | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 code/queryhdb/queryhdbstandard.q diff --git a/code/queryhdb/queryhdbstandard.q b/code/queryhdb/queryhdbstandard.q new file mode 100644 index 000000000..68cf3de2d --- /dev/null +++ b/code/queryhdb/queryhdbstandard.q @@ -0,0 +1,7 @@ +// reload function +reload:{ + .lg.o[`reload;"reloading QUERYHDB"]; + system"l ."} + +// Get the relevant QUERYHDB attributes +.proc.getattributes:{`date`tables!(@[value;`date;`date$()];tables[])} From 64aecc65be325ad0913dc9750b203e4754334f52 Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Tue, 14 Feb 2023 15:33:35 +0000 Subject: [PATCH 084/114] Add Historical fns --- code/querygateway/querygatewayfuncs.q | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index b0e126885..91f345889 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -243,3 +243,42 @@ NumberOfUsers:{ res:raze last .async.deferred[handle; query]; :res; }; + +PeakUsageHistorical:{[date] + users:GetUsers[]; + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:raze last .async.deferred[handle; query]; + + time:select distinct time from res; + + getquerycounts:{[res; users] ?[res; enlist (=; `u; enlist users); 0b; (enlist `queries)!(enlist `queries)]}[res; ]; + querycounts:getquerycounts'[users]; + + querycountsn:{x xcol y}'[users; querycounts]; + + getquerycountsnk:{[time; querycountsn] `time xkey ![querycountsn;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}[time; ]; + querycountsnk:getquerycountsnk'[querycountsn]; + + peakusage:0!(lj/)(querycountsnk); + + :update time:date + time from peakusage; + }; + +LongestRunningHistorical:{[date] + users:GetUsers[]; + query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", runtime=max runtime"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:raze last .async.deferred[handle; query]; + :ParseCmd res; + }; + +LongestRunningHeatMapHistorical:{[date] + users:GetUsers[]; + query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:raze last .async.deferred[handle; query]; + + :ParseCmd res; + }; + From 2df2ea5655b0a2b4c8c41d85a1544b9df961260e Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Wed, 15 Feb 2023 15:58:37 +0000 Subject: [PATCH 085/114] Filter by process --- code/querygateway/querygatewayfuncs.q | 55 +++++++++++++-------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 91f345889..784056724 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -118,28 +118,28 @@ ParseCmd:{[res] :remainder,' ([] func:f; query:q; proc:p); }; -QueryCountsRealtime:{ +QueryCountsRealtime:{[process] users:GetUsers[]; - query:"select count i from usage where u in ", (.Q.s1 users); + query:"select count i from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; }; -QueryUserCountsRealtime:{ +QueryUserCountsRealtime:{[process] users:GetUsers[]; - query:"select queries:count i by u from usage where u in ", (.Q.s1 users); + query:"select queries:count i by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; }; -QueryCountsHistorical:{[date] +QueryCountsHistorical:{[date;process] users:GetUsers[]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); - 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 users); + 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); // log error query:()] @@ -149,12 +149,12 @@ QueryCountsHistorical:{[date] :res; }; -QueryUserCountsHistorical:{[date] +QueryUserCountsHistorical:{[date;process] users:GetUsers[]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); - 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 users); + 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); // log error query:()] @@ -164,9 +164,9 @@ QueryUserCountsHistorical:{[date] :res; }; -PeakUsage:{ +PeakUsage:{[process] users:GetUsers[]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users); + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -185,17 +185,17 @@ PeakUsage:{ :update time:.z.d + time from peakusage; }; -LongestRunning:{ +LongestRunning:{[process] users:GetUsers[]; - query:"select time, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=max runtime"; + query:"select time, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :ParseCmd res; }; -LongestRunningHeatMap:{ +LongestRunningHeatMap:{[process] users:GetUsers[]; - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -203,22 +203,21 @@ LongestRunningHeatMap:{ }; //Return percentage of queries that were successful by user -QueryErrorPercentage:{ +QueryErrorPercentage:{[process] users:GetUsers[]; - query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where u in ", (.Q.s1 users); + query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; }; -QueryErrorPercentageHistorical:{[date] +QueryErrorPercentageHistorical:{[date;process] users:GetUsers[]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); - 2=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u in ", (.Q.s1 users); + 1=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 2=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); // log error query:()] - query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; :res; @@ -244,9 +243,9 @@ NumberOfUsers:{ :res; }; -PeakUsageHistorical:{[date] +PeakUsageHistorical:{[date;process] users:GetUsers[]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users); + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; @@ -265,17 +264,17 @@ PeakUsageHistorical:{[date] :update time:date + time from peakusage; }; -LongestRunningHistorical:{[date] +LongestRunningHistorical:{[date;process] users:GetUsers[]; - query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", runtime=max runtime"; + query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; :ParseCmd res; }; -LongestRunningHeatMapHistorical:{[date] +LongestRunningHeatMapHistorical:{[date;process] users:GetUsers[]; - query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; + query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; From ef40422e11328764e395132c89624d36d1ff1dc8 Mon Sep 17 00:00:00 2001 From: acreehay Date: Thu, 16 Feb 2023 04:22:43 +0000 Subject: [PATCH 086/114] extend users to ignore for query analytics --- code/handlers/logusage.q | 2 +- config/settings/default.q | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index 7d784b6f8..98ad31147 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -21,7 +21,7 @@ logtodisk:@[value;`logtodisk;1b] // whether to log to disk or not logtomemory:@[value;`logtomemory;1b] // write query logs to memory ignore:@[value;`ignore;1b] // check the ignore list for functions to ignore ignorelist:@[value;`ignorelist;(`upd;"upd")] // the list of functions to ignore -ignoreusers:@[value;`ignoreusers;`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway] // clients to ignore for usage monitoring +ignoreusers:@[value;`ignoreusers;()] // clients to ignore for query logging flushinterval:@[value;`flushinterval;0D00:30:00] // default value for how often to flush the in-memory logs flushtime:@[value;`flushtime;0D03] // default value for how long to persist the in-memory logs suppressalias:@[value;`suppressalias;0b] // whether to suppress the log file alias creation diff --git a/config/settings/default.q b/config/settings/default.q index e1ebe3047..7ce0c532f 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -20,7 +20,7 @@ logtodisk:1b // whether to log to disk or not logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls -ignoreusers:`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway // clients to ignore for usage monitoring +ignoreusers:raze (`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway`gateway`dqc`dqe;.z.u;`) // clients to ignore for query logging flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing suppressalias:0b // whether to suppress the log file alias creation From ae33013ffb6e33e4c8adb432514b2b7c1bd9461c Mon Sep 17 00:00:00 2001 From: acreehay Date: Thu, 16 Feb 2023 09:53:45 +0000 Subject: [PATCH 087/114] modified method of finding users across multiple procs --- code/querygateway/querygatewayfuncs.q | 33 ++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 784056724..11f1ee3de 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -96,6 +96,15 @@ GetDateRange:{[query] :eval each date; }; +//functions to set up variables +//RealtimeProcs:{}; + +//HistoricalProcs{}; + +//HistoricalDates:{}; + + + GetUsers:{ query:"first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`gateway; @@ -104,6 +113,24 @@ GetUsers:{ :res; }; +GetUsersRDB:{ + handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryrdb),":querygateway:pass"; + usageusers:handle"first flip select distinct u from usage"; + ignoreusers:`,`acreehay,`admin,exec distinct proctype from .servers.procstab; + res:usageusers except ignoreusers; + if[1=count res; :first res]; + :res; + }; + +GetUsersHDB:{[dt] + handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryhdb),":querygateway:pass"; + usageusers:handle"first flip select distinct u from usage where date=",string dt; + ignoreusers:`,`acreehay,`admin,exec distinct proctype from .servers.procstab; + res:usageusers except ignoreusers; + if[1=count res; :first res]; + :res; + }; + ParseCmd:{[res] cmd:raze value flip select cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; @@ -119,7 +146,7 @@ ParseCmd:{[res] }; QueryCountsRealtime:{[process] - users:GetUsers[]; + users:GetUsersRDB[]; query:"select count i from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -127,7 +154,7 @@ QueryCountsRealtime:{[process] }; QueryUserCountsRealtime:{[process] - users:GetUsers[]; + users:GetUsersRDB[]; query:"select queries:count i by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -204,7 +231,7 @@ LongestRunningHeatMap:{[process] //Return percentage of queries that were successful by user QueryErrorPercentage:{[process] - users:GetUsers[]; + users:GetUsersRDB[]; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; From d565476e3897b618e6bf6e1f4f0aa8c8e5780f98 Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Thu, 16 Feb 2023 18:44:24 +0000 Subject: [PATCH 088/114] Functions call getUsersRDB/HDB --- code/querygateway/querygatewayfuncs.q | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 11f1ee3de..c8f6467e7 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -162,7 +162,7 @@ QueryUserCountsRealtime:{[process] }; QueryCountsHistorical:{[date;process] - users:GetUsers[]; + users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); @@ -177,7 +177,7 @@ QueryCountsHistorical:{[date;process] }; QueryUserCountsHistorical:{[date;process] - users:GetUsers[]; + users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); @@ -192,7 +192,7 @@ QueryUserCountsHistorical:{[date;process] }; PeakUsage:{[process] - users:GetUsers[]; + users:GetUsersRDB[]; query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -213,7 +213,7 @@ PeakUsage:{[process] }; LongestRunning:{[process] - users:GetUsers[]; + users:GetUsersRDB[]; query:"select time, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -221,7 +221,7 @@ LongestRunning:{[process] }; LongestRunningHeatMap:{[process] - users:GetUsers[]; + users:GetUsersRDB[]; query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -239,7 +239,7 @@ QueryErrorPercentage:{[process] }; QueryErrorPercentageHistorical:{[date;process] - users:GetUsers[]; + users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error 1=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); 2=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); @@ -271,7 +271,7 @@ NumberOfUsers:{ }; PeakUsageHistorical:{[date;process] - users:GetUsers[]; + users:GetUsersHDB[date]; query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; @@ -292,7 +292,7 @@ PeakUsageHistorical:{[date;process] }; LongestRunningHistorical:{[date;process] - users:GetUsers[]; + users:GetUsersHDB[date]; query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; @@ -300,7 +300,7 @@ LongestRunningHistorical:{[date;process] }; LongestRunningHeatMapHistorical:{[date;process] - users:GetUsers[]; + users:GetUsersHDB[date]; query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; From 0bbbd7d1822c416bf9259ca023d06aec52b2c92a Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Wed, 22 Feb 2023 17:36:00 +0000 Subject: [PATCH 089/114] update functions for grafana --- code/querygateway/querygatewayfuncs.q | 68 +++++++++++++++++++-------- 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index c8f6467e7..75addae46 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -116,7 +116,7 @@ GetUsers:{ GetUsersRDB:{ handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryrdb),":querygateway:pass"; usageusers:handle"first flip select distinct u from usage"; - ignoreusers:`,`acreehay,`admin,exec distinct proctype from .servers.procstab; + ignoreusers:`,`sboyd,`admin,exec distinct proctype from .servers.procstab; res:usageusers except ignoreusers; if[1=count res; :first res]; :res; @@ -125,7 +125,7 @@ GetUsersRDB:{ GetUsersHDB:{[dt] handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryhdb),":querygateway:pass"; usageusers:handle"first flip select distinct u from usage where date=",string dt; - ignoreusers:`,`acreehay,`admin,exec distinct proctype from .servers.procstab; + ignoreusers:`,`sboyd,`admin,exec distinct proctype from .servers.procstab; res:usageusers except ignoreusers; if[1=count res; :first res]; :res; @@ -203,6 +203,7 @@ PeakUsage:{[process] querycounts:getquerycounts'[users]; querycountsn:{x xcol y}'[users; querycounts]; + querycountsn:querycountsn where not 0=count each querycountsn; getquerycountsnk:{[time; querycountsn] `time xkey ![querycountsn;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}[time; ]; querycountsnk:getquerycountsnk'[querycountsn]; @@ -212,13 +213,13 @@ PeakUsage:{[process] :update time:.z.d + time from peakusage; }; -LongestRunning:{[process] - users:GetUsersRDB[]; - query:"select time, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:raze last .async.deferred[handle; query]; - :ParseCmd res; - }; +//LongestRunning:{[process] +// users:GetUsersRDB[]; +// query:"select time, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; +// handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; +// res:raze last .async.deferred[handle; query]; +// :ParseCmd res; +// }; LongestRunningHeatMap:{[process] users:GetUsersRDB[]; @@ -230,19 +231,47 @@ LongestRunningHeatMap:{[process] }; //Return percentage of queries that were successful by user +//QueryErrorPercentage:{[process] +// users:GetUsersRDB[]; +// query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); +// handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; +// res:raze last .async.deferred[handle; query]; +// :res; +// }; + QueryErrorPercentage:{[process] users:GetUsersRDB[]; - query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + query:"select completed:count i where status=\"c\", error:count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; }; +LongestRunning:{[process] + users:GetUsersRDB[]; + query:"select max runtime by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:raze last .async.deferred[handle; query]; + :res; + }; + +LongestRunningHistorical:{[date;process] + users:GetUsersHDB[date]; + $[.z.d<=date; query:(); // log error + 1=count date; query:"select max runtime by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 2=count date; query:"select max runtime by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + // log error + query:()] + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:raze last .async.deferred[handle; query]; + :res; + }; + QueryErrorPercentageHistorical:{[date;process] users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); - 2=count date; query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 1=count date; query:"select completed:count i where status=\"c\", error:count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 2=count date; query:"select completed:count i where status=\"c\", error:count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; @@ -282,6 +311,7 @@ PeakUsageHistorical:{[date;process] querycounts:getquerycounts'[users]; querycountsn:{x xcol y}'[users; querycounts]; + querycountsn:querycountsn where not 0=count each querycountsn; getquerycountsnk:{[time; querycountsn] `time xkey ![querycountsn;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}[time; ]; querycountsnk:getquerycountsnk'[querycountsn]; @@ -291,13 +321,13 @@ PeakUsageHistorical:{[date;process] :update time:date + time from peakusage; }; -LongestRunningHistorical:{[date;process] - users:GetUsersHDB[date]; - query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - res:raze last .async.deferred[handle; query]; - :ParseCmd res; - }; +//LongestRunningHistorical:{[date;process] +// users:GetUsersHDB[date]; +// query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; +// handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; +// res:raze last .async.deferred[handle; query]; +// :ParseCmd res; +// }; LongestRunningHeatMapHistorical:{[date;process] users:GetUsersHDB[date]; From 6285ef645f0806bbd5397483287d8d43d4e9851e Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Fri, 24 Feb 2023 16:05:04 +0000 Subject: [PATCH 090/114] Dashboard improvements --- code/querygateway/querygatewayfuncs.q | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 75addae46..d4b956b85 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -116,7 +116,7 @@ GetUsers:{ GetUsersRDB:{ handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryrdb),":querygateway:pass"; usageusers:handle"first flip select distinct u from usage"; - ignoreusers:`,`sboyd,`admin,exec distinct proctype from .servers.procstab; + ignoreusers:`,(`$system"echo $USER"),`admin,exec distinct proctype from .servers.procstab; res:usageusers except ignoreusers; if[1=count res; :first res]; :res; @@ -125,7 +125,7 @@ GetUsersRDB:{ GetUsersHDB:{[dt] handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryhdb),":querygateway:pass"; usageusers:handle"first flip select distinct u from usage where date=",string dt; - ignoreusers:`,`sboyd,`admin,exec distinct proctype from .servers.procstab; + ignoreusers:`,(`$system"echo $USER"),`admin,exec distinct proctype from .servers.procstab; res:usageusers except ignoreusers; if[1=count res; :first res]; :res; @@ -196,6 +196,7 @@ PeakUsage:{[process] query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; +// delete from `res where time=`minute$.z.t; time:select distinct time from res; From c1e2072080f185d32479da5ddf79ea3b8cd91104 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Wed, 1 Mar 2023 18:35:21 +0000 Subject: [PATCH 091/114] Update PeakUsage fn to handle null data --- code/querygateway/querygatewayfuncs.q | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index d4b956b85..5b8852dec 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -196,20 +196,14 @@ PeakUsage:{[process] query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; -// delete from `res where time=`minute$.z.t; - time:select distinct time from res; - - getquerycounts:{[res; users] ?[res; enlist (=; `u; enlist users); 0b; (enlist `queries)!(enlist `queries)]}[res; ]; + // select separate table of times and queries for each user + getquerycounts:{[res; users] ?[res; enlist(=; `u; `users); 0b; (`time`queries)!(`time`queries)]}[res; ]; querycounts:getquerycounts'[users]; + // rename 'queries' col with name of user for each table + querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; - querycountsn:{x xcol y}'[users; querycounts]; - querycountsn:querycountsn where not 0=count each querycountsn; - - getquerycountsnk:{[time; querycountsn] `time xkey ![querycountsn;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}[time; ]; - querycountsnk:getquerycountsnk'[querycountsn]; - - peakusage:0!(lj/)(querycountsnk); + peakusage:0!(pj/)1!'querycountsn; :update time:.z.d + time from peakusage; }; From 4c416296b5473c86e2eff77f50bc9fa978b3f650 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 2 Mar 2023 11:32:06 +0000 Subject: [PATCH 092/114] Clean up CmdParse --- code/querygateway/querygatewayfuncs.q | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 5b8852dec..8bfb6e2d1 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -134,15 +134,14 @@ GetUsersHDB:{[dt] ParseCmd:{[res] cmd:raze value flip select cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - reslist:select cmd from update cmd:";" vs ' cmd from res; + cmdsplit:select cmd from update cmd:";" vs ' cmd from res; - // grab function, query and process and parse out any - // brackets or slashes in strings - f:`$1_'first each first value flip reslist; - q:1_'-1_'first each next each first value flip reslist; - p:`$-1_'first each next each next each first value flip reslist; + // split cmd into three columns + cmdcolsplit:select func, query, proc from @[cmdsplit; `func`query`proc; :; flip cmdsplit`cmd]; + // parse out unwanted chars + cmdcolsplitparsed:update func:`$2_'func, query:1_'-1_'query, proc:`$1_'-1_'proc from cmdcolsplit; - :remainder,' ([] func:f; query:q; proc:p); + :remainder,'cmdcolsplitparsed; }; QueryCountsRealtime:{[process] From 59d4f4b272650bf185d33280d2b0de1abbe2c153 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 2 Mar 2023 15:03:12 +0000 Subject: [PATCH 093/114] Remove duplicates by restricting analytics to completed queries --- code/querygateway/querygatewayfuncs.q | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 8bfb6e2d1..ce1b82bd1 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -146,7 +146,7 @@ ParseCmd:{[res] QueryCountsRealtime:{[process] users:GetUsersRDB[]; - query:"select count i from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + query:"select count i from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; @@ -154,7 +154,7 @@ QueryCountsRealtime:{[process] QueryUserCountsRealtime:{[process] users:GetUsersRDB[]; - query:"select queries:count i by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + query:"select queries:count i by u from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; @@ -164,8 +164,8 @@ QueryCountsHistorical:{[date;process] users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); - 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "),", ", status=", (.Q.s1 "c"), "procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); // log error query:()] @@ -179,8 +179,8 @@ QueryUserCountsHistorical:{[date;process] users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); - 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), "u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); + 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), "procname in ", (.Q.s1 process); // log error query:()] @@ -192,7 +192,7 @@ QueryUserCountsHistorical:{[date;process] PeakUsage:{[process] users:GetUsersRDB[]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -217,7 +217,7 @@ PeakUsage:{[process] LongestRunningHeatMap:{[process] users:GetUsersRDB[]; - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), " runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; @@ -243,7 +243,7 @@ QueryErrorPercentage:{[process] LongestRunning:{[process] users:GetUsersRDB[]; - query:"select max runtime by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + query:"select max runtime by u from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; :res; @@ -252,8 +252,8 @@ LongestRunning:{[process] LongestRunningHistorical:{[date;process] users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select max runtime by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); - 2=count date; query:"select max runtime by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 1=count date; query:"select max runtime by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); + 2=count date; query:"select max runtime by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), " , procname in ", (.Q.s1 process); // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; @@ -295,7 +295,7 @@ NumberOfUsers:{ PeakUsageHistorical:{[date;process] users:GetUsersHDB[date]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; @@ -325,7 +325,7 @@ PeakUsageHistorical:{[date;process] LongestRunningHeatMapHistorical:{[date;process] users:GetUsersHDB[date]; - query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=(max; runtime) fby 10 xbar time.minute"; + query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; From 18b5c05d372f046b80560d9bfbfbbfad88906908 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 2 Mar 2023 15:15:37 +0000 Subject: [PATCH 094/114] Fix typo --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index ce1b82bd1..480cb2f10 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -217,7 +217,7 @@ PeakUsage:{[process] LongestRunningHeatMap:{[process] users:GetUsersRDB[]; - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), " runtime=(max; runtime) fby 10 xbar time.minute"; + query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), ", runtime=(max; runtime) fby 10 xbar time.minute"; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; From b8b707a8d3e2c1141a54cee9948ec2f7e96f34fe Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Thu, 2 Mar 2023 16:10:11 +0000 Subject: [PATCH 095/114] Fix typo --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 480cb2f10..1e7c3a9ea 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -179,7 +179,7 @@ QueryUserCountsHistorical:{[date;process] users:GetUsersHDB[date]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), "u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); + 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), "procname in ", (.Q.s1 process); // log error query:()] From 9f20891c04d8881299e4165f077a83ef84950cbf Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Mar 2023 11:07:12 +0000 Subject: [PATCH 096/114] Overhaul proc picker functionality --- code/handlers/logusage.q | 1 + code/querygateway/querygatewayfuncs.q | 41 +++++++++++++++++++++++---- config/settings/default.q | 1 + 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index 98ad31147..98f69f106 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -21,6 +21,7 @@ logtodisk:@[value;`logtodisk;1b] // whether to log to disk or not logtomemory:@[value;`logtomemory;1b] // write query logs to memory ignore:@[value;`ignore;1b] // check the ignore list for functions to ignore ignorelist:@[value;`ignorelist;(`upd;"upd")] // the list of functions to ignore +allowedusers:@[value;`allowedusers;()] ignoreusers:@[value;`ignoreusers;()] // clients to ignore for query logging flushinterval:@[value;`flushinterval;0D00:30:00] // default value for how often to flush the in-memory logs flushtime:@[value;`flushtime;0D03] // default value for how long to persist the in-memory logs diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 1e7c3a9ea..5e4828ea0 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -144,12 +144,41 @@ ParseCmd:{[res] :remainder,'cmdcolsplitparsed; }; -QueryCountsRealtime:{[process] - users:GetUsersRDB[]; - query:"select count i from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:raze last .async.deferred[handle; query]; - :res; +ParseCmdProcs:{[res] + cmdsplit:select cmd:-2#'";" vs/: cmd from res; + remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; + + cmdcolsplit:select user, query from @[cmdsplit; `user`query; :; flip cmdsplit`cmd]; + cmdcolsplitparsed:update user:`$1_'user, query:1_'-3_'query from cmdcolsplit; + + :remainder,'cmdcolsplitparsed; + }; + +ProcPickerRDB:{[process] + $[process=`any; + phrase:"proctype=`rdb"; + phrase:"procname=", .Q.s1 process]; + + :phrase; + }; + +ProcPickerHDB:{[process] + $[process=`any; + phrase:"proctype=`hdb"; + phrase:"procname", .Q.s1 process]; + + :phrase; + }; + +QueryCountsRealtime:{[process] + users:GetUsersRDB[]; + procphrase:ProcPickerRDB[process]; + + query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:raze last .async.deferred[handle; query]; + + :select count i from ParseCmdProcs[res] where user in .usage.allowedusers; }; QueryUserCountsRealtime:{[process] diff --git a/config/settings/default.q b/config/settings/default.q index 7ce0c532f..fb0e8341b 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -20,6 +20,7 @@ logtodisk:1b // whether to log to disk or not logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls +allowedusers:`angus`michael`stephen ignoreusers:raze (`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway`gateway`dqc`dqe;.z.u;`) // clients to ignore for query logging flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing From eff557558e33c1f767cab476b6e45ae7f83c3bb5 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Mar 2023 12:00:35 +0000 Subject: [PATCH 097/114] Expand ParseCmd --- code/querygateway/querygatewayfuncs.q | 39 +++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 5e4828ea0..b5c4677fb 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -131,27 +131,26 @@ GetUsersHDB:{[dt] :res; }; -ParseCmd:{[res] - cmd:raze value flip select cmd from res; - remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - cmdsplit:select cmd from update cmd:";" vs ' cmd from res; +ParseCmd:{[res; procpicker] + $[procpicker; - // split cmd into three columns - cmdcolsplit:select func, query, proc from @[cmdsplit; `func`query`proc; :; flip cmdsplit`cmd]; - // parse out unwanted chars - cmdcolsplitparsed:update func:`$2_'func, query:1_'-1_'query, proc:`$1_'-1_'proc from cmdcolsplit; + [cmdsplit:select cmd:-2#'";" vs/: cmd from res; + remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - :remainder,'cmdcolsplitparsed; - }; + cmdcolsplit:select user, query from @[cmdsplit; `user`query; :; flip cmdsplit`cmd]; + cmdcolsplitparsed:update user:`$1_'user, query:1_'-3_'query from cmdcolsplit; + + :remainder,'cmdcolsplitparsed;]; -ParseCmdProcs:{[res] - cmdsplit:select cmd:-2#'";" vs/: cmd from res; - remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; + [cmdsplit:select cmd from update cmd:";" vs ' cmd from res; + remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - cmdcolsplit:select user, query from @[cmdsplit; `user`query; :; flip cmdsplit`cmd]; - cmdcolsplitparsed:update user:`$1_'user, query:1_'-3_'query from cmdcolsplit; + // split cmd into three columns + cmdcolsplit:select func, query, proc from @[cmdsplit; `func`query`proc; :; flip cmdsplit`cmd]; + // parse out unwanted chars + cmdcolsplitparsed:update func:`$2_'func, query:1_'-1_'query, proc:`$1_'-1_'proc from cmdcolsplit; - :remainder,'cmdcolsplitparsed; + :remainder,'cmdcolsplitparsed;] ]; }; ProcPickerRDB:{[process] @@ -172,13 +171,13 @@ ProcPickerHDB:{[process] QueryCountsRealtime:{[process] users:GetUsersRDB[]; - procphrase:ProcPickerRDB[process]; + procphrase:ProcPickerRDB[`$process]; query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :select count i from ParseCmdProcs[res] where user in .usage.allowedusers; + :select count i from ParseCmd[res; 1b] where user in .usage.allowedusers; }; QueryUserCountsRealtime:{[process] @@ -250,7 +249,7 @@ LongestRunningHeatMap:{[process] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :ParseCmd res; + :ParseCmd[res; 0b]; }; //Return percentage of queries that were successful by user @@ -358,6 +357,6 @@ LongestRunningHeatMapHistorical:{[date;process] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; - :ParseCmd res; + :ParseCmd[res; 0b]; }; From 11da8400e3db868bb384f1e75dcdb2df7cd298be Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Mar 2023 14:34:03 +0000 Subject: [PATCH 098/114] Begin extending proc picker logic to other fns --- code/querygateway/querygatewayfuncs.q | 35 +++++++++++++++------------ 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index b5c4677fb..4fb03f428 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -122,9 +122,9 @@ GetUsersRDB:{ :res; }; -GetUsersHDB:{[dt] +GetUsersHDB:{[date] handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryhdb),":querygateway:pass"; - usageusers:handle"first flip select distinct u from usage where date=",string dt; + usageusers:handle"first flip select distinct u from usage where date=",string date; ignoreusers:`,(`$system"echo $USER"),`admin,exec distinct proctype from .servers.procstab; res:usageusers except ignoreusers; if[1=count res; :first res]; @@ -137,8 +137,8 @@ ParseCmd:{[res; procpicker] [cmdsplit:select cmd:-2#'";" vs/: cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - cmdcolsplit:select user, query from @[cmdsplit; `user`query; :; flip cmdsplit`cmd]; - cmdcolsplitparsed:update user:`$1_'user, query:1_'-3_'query from cmdcolsplit; + cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; + cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; :remainder,'cmdcolsplitparsed;]; @@ -177,45 +177,50 @@ QueryCountsRealtime:{[process] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :select count i from ParseCmd[res; 1b] where user in .usage.allowedusers; + :select count i from ParseCmd[res; 1b] where originaluser in users; }; QueryUserCountsRealtime:{[process] users:GetUsersRDB[]; - query:"select queries:count i by u from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); + procphrase:ProcPickerRDB[`$process]; + + query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :res; + + :select queries:count i by originaluser from ParseCmd[res; 1b] where originaluser in users; }; -QueryCountsHistorical:{[date;process] +QueryCountsHistorical:{[date; process] users:GetUsersHDB[date]; + procphrase:ProcPickerHDB[`$process] $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); - 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "),", ", status=", (.Q.s1 "c"), "procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + 1=count date; query:"select from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", ", procphrase; + 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "),", ", status=", (.Q.s1 "c"), ", ", procphrase; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; raze query]; - :res; + :select queries:count i from ParseCmd[res; 1b] where originaluser in users; }; -QueryUserCountsHistorical:{[date;process] +QueryUserCountsHistorical:{[date; process] users:GetUsersHDB[date]; + procphrase:ProcPickerHDB[`$process]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select queries:count i by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); - 2=count date; query:"select queries:count i by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), "procname in ", (.Q.s1 process); + 1=count date; query:"select from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", ", procphrase; + 2=count date; query:"select from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", status=", (.Q.s1 "c"), procphrase; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; raze query]; - :res; + :select queries:count i by originaluser from ParseCmd[res; 1b] where originaluser in users; }; PeakUsage:{[process] From 9f0763a597739d0c77f1cd34c47c8fca8e8a5d18 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Fri, 3 Mar 2023 14:42:00 +0000 Subject: [PATCH 099/114] Fix typos --- code/querygateway/querygatewayfuncs.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 4fb03f428..e3b801b28 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -193,7 +193,7 @@ QueryUserCountsRealtime:{[process] QueryCountsHistorical:{[date; process] users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process] + procphrase:ProcPickerHDB[`$process]; $[.z.d<=date; query:(); // log error 1=count date; query:"select from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", ", procphrase; @@ -213,7 +213,7 @@ QueryUserCountsHistorical:{[date; process] $[.z.d<=date; query:(); // log error 1=count date; query:"select from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", ", procphrase; - 2=count date; query:"select from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", status=", (.Q.s1 "c"), procphrase; + 2=count date; query:"select from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", status=", (.Q.s1 "c"), ", ", procphrase; // log error query:()] From 7af4c0d18248301a5dc17c06c7d0f653254cb86f Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 10:48:26 +0000 Subject: [PATCH 100/114] Complete proc picker implementation --- code/querygateway/querygatewayfuncs.q | 154 ++++++++++++++------------ 1 file changed, 86 insertions(+), 68 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index e3b801b28..743f22fa3 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -131,30 +131,19 @@ GetUsersHDB:{[date] :res; }; -ParseCmd:{[res; procpicker] - $[procpicker; +// currently setup to deal with ubiquitous error sting in cmd +// will need updated when the foregoing is fixed +ParseCmd:{[res] + cmdsplit:select cmd:-2#'";" vs/: cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - [cmdsplit:select cmd:-2#'";" vs/: cmd from res; - remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; + cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; + cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; - cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; - cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; - - :remainder,'cmdcolsplitparsed;]; - - [cmdsplit:select cmd from update cmd:";" vs ' cmd from res; - remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; - - // split cmd into three columns - cmdcolsplit:select func, query, proc from @[cmdsplit; `func`query`proc; :; flip cmdsplit`cmd]; - // parse out unwanted chars - cmdcolsplitparsed:update func:`$2_'func, query:1_'-1_'query, proc:`$1_'-1_'proc from cmdcolsplit; - - :remainder,'cmdcolsplitparsed;] ]; + :remainder,'cmdcolsplitparsed; }; ProcPickerRDB:{[process] - $[process=`any; + $[process=`all; phrase:"proctype=`rdb"; phrase:"procname=", .Q.s1 process]; @@ -162,9 +151,9 @@ ProcPickerRDB:{[process] }; ProcPickerHDB:{[process] - $[process=`any; + $[process=`all; phrase:"proctype=`hdb"; - phrase:"procname", .Q.s1 process]; + phrase:"procname=", .Q.s1 process]; :phrase; }; @@ -177,7 +166,7 @@ QueryCountsRealtime:{[process] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :select count i from ParseCmd[res; 1b] where originaluser in users; + :select count i from ParseCmd[res] where originaluser in users; }; QueryUserCountsRealtime:{[process] @@ -188,7 +177,7 @@ QueryUserCountsRealtime:{[process] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :select queries:count i by originaluser from ParseCmd[res; 1b] where originaluser in users; + :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; }; QueryCountsHistorical:{[date; process] @@ -196,15 +185,15 @@ QueryCountsHistorical:{[date; process] procphrase:ProcPickerHDB[`$process]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", ", procphrase; - 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "),", ", status=", (.Q.s1 "c"), ", ", procphrase; + 1=count date; query:"select from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; + 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "),", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; raze query]; - :select queries:count i from ParseCmd[res; 1b] where originaluser in users; + :select queries:count i from ParseCmd[res] where originaluser in users; }; QueryUserCountsHistorical:{[date; process] @@ -212,25 +201,29 @@ QueryUserCountsHistorical:{[date; process] procphrase:ProcPickerHDB[`$process]; $[.z.d<=date; query:(); // log error - 1=count date; query:"select from usage where date=", (.Q.s1 date), ", status=", (.Q.s1 "c"), ", ", procphrase; - 2=count date; query:"select from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", status=", (.Q.s1 "c"), ", ", procphrase; + 1=count date; query:"select from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; + 2=count date; query:"select from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; raze query]; - :select queries:count i by originaluser from ParseCmd[res; 1b] where originaluser in users; + :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; }; PeakUsage:{[process] users:GetUsersRDB[]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); + procphrase:ProcPickerRDB[`$process]; + + query:"select time, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - // select separate table of times and queries for each user - getquerycounts:{[res; users] ?[res; enlist(=; `u; `users); 0b; (`time`queries)!(`time`queries)]}[res; ]; + resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; + + // select separate tables of times and queries for each user + getquerycounts:{[resparsed; users] ?[resparsed; enlist(=; `origingaluser; `users); 0b; (`time`queries)!(`time`queries)]}[res; ]; querycounts:getquerycounts'[users]; // rename 'queries' col with name of user for each table querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; @@ -249,12 +242,15 @@ PeakUsage:{[process] // }; LongestRunningHeatMap:{[process] - users:GetUsersRDB[]; - query:"select time:.z.d + 10 xbar time.minute, runtime, u, cmd from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), ", runtime=(max; runtime) fby 10 xbar time.minute"; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:raze last .async.deferred[handle; query]; - - :ParseCmd[res; 0b]; + users:GetUsersRDB[]; + procphrase:ProcPickerRDB[`$process]; + + query:"select time, runtime, proctype, procname, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + res:raze last .async.deferred[handle; query]; + resparsed:ParseCmd[res]; + + :select time:.z.d + 10 xbar time.minute, runtime, proctype, procname, originaluser, query from resparsed where originaluser in users, runtime=(max; runtime) fby 10 xbar time.minute; }; //Return percentage of queries that were successful by user @@ -268,42 +264,62 @@ LongestRunningHeatMap:{[process] QueryErrorPercentage:{[process] users:GetUsersRDB[]; - query:"select completed:count i where status=\"c\", error:count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + procphrase:ProcPickerRDB[`$process]; + + // where status is "c" or "e" + query:"select status, cmd from usage where u=`gateway, status in ", (string `ce), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :res; + + resparsed:ParseCmd[res]; + + :select completed:count i where status="c", error:count i where status="e" from resparsed where originaluser in users; }; LongestRunning:{[process] users:GetUsersRDB[]; - query:"select max runtime by u from usage where u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); + procphrase:ProcPickerRDB[`$process]; + + query:"select runtime, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; res:raze last .async.deferred[handle; query]; - :res; + resparsed:ParseCmd[res]; + + :select max runtime by originaluser from resparsed where originaluser in users; }; -LongestRunningHistorical:{[date;process] +LongestRunningHistorical:{[date; process] users:GetUsersHDB[date]; + procphrase:ProcPickerHDB[`$process]; + $[.z.d<=date; query:(); // log error - 1=count date; query:"select max runtime by u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); - 2=count date; query:"select max runtime by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), " , procname in ", (.Q.s1 process); + 1=count date; query:"select runtime, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; + 2=count date; query:"select runtime, cmd from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; - :res; + resparsed:ParseCmd[res]; + + :select max runtime by originaluser from resparsed where originaluser in users; }; -QueryErrorPercentageHistorical:{[date;process] +QueryErrorPercentageHistorical:{[date; process] users:GetUsersHDB[date]; + procphrase:ProcPickerHDB[`$process]; + $[.z.d<=date; query:(); // log error - 1=count date; query:"select completed:count i where status=\"c\", error:count i where status=\"e\" by u from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); - 2=count date; query:"select completed:count i where status=\"c\", error:count i where status=\"e\" by u from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); + // where status is "c" or "e" + 1=count date; query:"select status, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status in ", (string `ce), ", ", procphrase; + // where status is "c" or "e" + 2=count date; query:"select status, cmd from usage where date in (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u=`gateway, status in ", (string `ce), ", ", procphrase; // log error query:()] handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; - :res; + resparsed:ParseCmd:[res]; + + :select completed:count i where status="c", error:count i where status="e" from resparsed where originaluser in users; }; //Return queries which take longer than given runtime input, t @@ -326,26 +342,25 @@ NumberOfUsers:{ :res; }; -PeakUsageHistorical:{[date;process] - users:GetUsersHDB[date]; - query:"`time xcol 0!select queries:count i by 10 xbar time.minute, u from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process); +PeakUsageHistorical:{[date; process] + users:GetUsersHDB[]; + procphrase:ProcPickerHDB[`$process]; + + query:"select time, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; res:raze last .async.deferred[handle; query]; - time:select distinct time from res; + resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; - getquerycounts:{[res; users] ?[res; enlist (=; `u; enlist users); 0b; (enlist `queries)!(enlist `queries)]}[res; ]; + // select separate tables of times and queries for each user + getquerycounts:{[resparsed; users] ?[resparsed; enlist(=; `origingaluser; `users); 0b; (`time`queries)!(`time`queries)]}[res; ]; querycounts:getquerycounts'[users]; + // rename 'queries' col with name of user for each table + querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; - querycountsn:{x xcol y}'[users; querycounts]; - querycountsn:querycountsn where not 0=count each querycountsn; - - getquerycountsnk:{[time; querycountsn] `time xkey ![querycountsn;();0b;(enlist `time)!enlist (raze; (each; raze; `time))]}[time; ]; - querycountsnk:getquerycountsnk'[querycountsn]; - - peakusage:0!(lj/)(querycountsnk); + peakusage:0!(pj/)1!'querycountsn; - :update time:date + time from peakusage; + :update time:.z.d + time from peakusage; }; //LongestRunningHistorical:{[date;process] @@ -356,12 +371,15 @@ PeakUsageHistorical:{[date;process] // :ParseCmd res; // }; -LongestRunningHeatMapHistorical:{[date;process] +LongestRunningHeatMapHistorical:{[date; process] users:GetUsersHDB[date]; - query:"select time:date + 10 xbar time.minute, runtime, u, cmd from usage where date=", (.Q.s1 date), ", u in ", (.Q.s1 users), ", status=", (.Q.s1 "c"), ", procname in ", (.Q.s1 process), ", runtime=(max; runtime) fby 10 xbar time.minute"; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; - res:raze last .async.deferred[handle; query]; + procphrase:ProcPickerHdb[`$process]; - :ParseCmd[res; 0b]; + query:"select time, runtime, proctype, procname, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; + handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + res:raze last .async.deferred[handle; query]; + resparsed:ParseCmd[res]; + + :select time:.z.d + 10 xbar time.minute, runtime, proctype, procname, originaluser, query from resparsed where originaluser in users, runtime=(max; runtime) fby 10 xbar time.minute; }; From 4ed74e61cdd323ab9cf590ec809280624d1e3acb Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 11:40:10 +0000 Subject: [PATCH 101/114] Fix wrong paramaters --- code/querygateway/querygatewayfuncs.q | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 743f22fa3..f735dd0e9 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -134,7 +134,9 @@ GetUsersHDB:{[date] // currently setup to deal with ubiquitous error sting in cmd // will need updated when the foregoing is fixed ParseCmd:{[res] - cmdsplit:select cmd:-2#'";" vs/: cmd from res; remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; + cmdsplit:select cmd:-2#'";" vs/: cmd from res; + /remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; + remainder:select from (cols[res] except `cmd)#res; cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; @@ -223,7 +225,7 @@ PeakUsage:{[process] resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; // select separate tables of times and queries for each user - getquerycounts:{[resparsed; users] ?[resparsed; enlist(=; `origingaluser; `users); 0b; (`time`queries)!(`time`queries)]}[res; ]; + getquerycounts:{[resparsed; users] ?[resparsed; enlist(in; `originaluser; `users); 0b; (`time`queries)!(`time`queries)]}[resparsed; ]; querycounts:getquerycounts'[users]; // rename 'queries' col with name of user for each table querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; @@ -343,7 +345,7 @@ NumberOfUsers:{ }; PeakUsageHistorical:{[date; process] - users:GetUsersHDB[]; + users:GetUsersHDB[date]; procphrase:ProcPickerHDB[`$process]; query:"select time, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; @@ -353,7 +355,7 @@ PeakUsageHistorical:{[date; process] resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; // select separate tables of times and queries for each user - getquerycounts:{[resparsed; users] ?[resparsed; enlist(=; `origingaluser; `users); 0b; (`time`queries)!(`time`queries)]}[res; ]; + getquerycounts:{[resparsed; users] ?[resparsed; enlist(in; `originaluser; `users); 0b; (`time`queries)!(`time`queries)]}[resparsed; ]; querycounts:getquerycounts'[users]; // rename 'queries' col with name of user for each table querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; @@ -373,7 +375,7 @@ PeakUsageHistorical:{[date; process] LongestRunningHeatMapHistorical:{[date; process] users:GetUsersHDB[date]; - procphrase:ProcPickerHdb[`$process]; + procphrase:ProcPickerHDB[`$process]; query:"select time, runtime, proctype, procname, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; @@ -382,4 +384,3 @@ LongestRunningHeatMapHistorical:{[date; process] :select time:.z.d + 10 xbar time.minute, runtime, proctype, procname, originaluser, query from resparsed where originaluser in users, runtime=(max; runtime) fby 10 xbar time.minute; }; - From 0e92e7ce86c10fd8245ecfad97bbad284597a274 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 13:14:24 +0000 Subject: [PATCH 102/114] Add GetHandle fn --- code/querygateway/querygatewayfuncs.q | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index f735dd0e9..da8b0d8cc 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -131,11 +131,14 @@ GetUsersHDB:{[date] :res; }; +GetHandle:{[proc] + :first -1?exec handle from .gw.availableserverstable[1b] where servertype=proc; + }; + // currently setup to deal with ubiquitous error sting in cmd // will need updated when the foregoing is fixed ParseCmd:{[res] cmdsplit:select cmd:-2#'";" vs/: cmd from res; - /remainder:update runtime:.proc.cd[] + runtime from (cols[res] except `cmd)#res; remainder:select from (cols[res] except `cmd)#res; cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; @@ -165,7 +168,7 @@ QueryCountsRealtime:{[process] procphrase:ProcPickerRDB[`$process]; query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; :select count i from ParseCmd[res] where originaluser in users; @@ -176,7 +179,7 @@ QueryUserCountsRealtime:{[process] procphrase:ProcPickerRDB[`$process]; query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; @@ -192,7 +195,7 @@ QueryCountsHistorical:{[date; process] // log error query:()] - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; raze query]; :select queries:count i from ParseCmd[res] where originaluser in users; @@ -208,7 +211,7 @@ QueryUserCountsHistorical:{[date; process] // log error query:()] - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; raze query]; :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; @@ -219,7 +222,7 @@ PeakUsage:{[process] procphrase:ProcPickerRDB[`$process]; query:"select time, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; @@ -248,7 +251,7 @@ LongestRunningHeatMap:{[process] procphrase:ProcPickerRDB[`$process]; query:"select time, runtime, proctype, procname, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd[res]; @@ -270,7 +273,7 @@ QueryErrorPercentage:{[process] // where status is "c" or "e" query:"select status, cmd from usage where u=`gateway, status in ", (string `ce), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd[res]; @@ -283,7 +286,7 @@ LongestRunning:{[process] procphrase:ProcPickerRDB[`$process]; query:"select runtime, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; + handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd[res]; @@ -299,7 +302,7 @@ LongestRunningHistorical:{[date; process] 2=count date; query:"select runtime, cmd from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; // log error query:()] - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd[res]; @@ -317,7 +320,7 @@ QueryErrorPercentageHistorical:{[date; process] 2=count date; query:"select status, cmd from usage where date in (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u=`gateway, status in ", (string `ce), ", ", procphrase; // log error query:()] - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd:[res]; @@ -349,7 +352,7 @@ PeakUsageHistorical:{[date; process] procphrase:ProcPickerHDB[`$process]; query:"select time, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; query]; resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; @@ -378,7 +381,7 @@ LongestRunningHeatMapHistorical:{[date; process] procphrase:ProcPickerHDB[`$process]; query:"select time, runtime, proctype, procname, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd[res]; From 2867e4c8d514833e7df8c4290ff7f05589c0904e Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 13:45:46 +0000 Subject: [PATCH 103/114] Error trap CmdParse --- code/querygateway/querygatewayfuncs.q | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index da8b0d8cc..a9713610b 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -138,12 +138,14 @@ GetHandle:{[proc] // currently setup to deal with ubiquitous error sting in cmd // will need updated when the foregoing is fixed ParseCmd:{[res] - cmdsplit:select cmd:-2#'";" vs/: cmd from res; - remainder:select from (cols[res] except `cmd)#res; + cmdsplit:@[{select cmd:-2#'";" vs/: cmd from x}; res; {.lg.e[`ParseCmd; "cmd col missing from paramater res, returning input and exiting function"]; :res;}]; + remainder:@[{select from (cols[res] except `cmd)#x}; res; ()]; cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; + if [0=count remainder; :cmdcolsplitparsed;]; + :remainder,'cmdcolsplitparsed; }; From afafe922047cba326a096d82211721cf40448394 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 13:59:36 +0000 Subject: [PATCH 104/114] Fix GetUsers functions --- code/querygateway/querygatewayfuncs.q | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index a9713610b..d16aa7b8e 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -114,21 +114,23 @@ GetUsers:{ }; GetUsersRDB:{ - handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryrdb),":querygateway:pass"; - usageusers:handle"first flip select distinct u from usage"; - ignoreusers:`,(`$system"echo $USER"),`admin,exec distinct proctype from .servers.procstab; - res:usageusers except ignoreusers; - if[1=count res; :first res]; - :res; + query:"select cmd from usage where u=`gateway"; + handle:GetHandle `queryrdb; + res:raze last .async.deferred[handle; query]; + resparsed:ParseCmd res; + + users:first value flip select distinct originaluser from resparsed; + :realusers:users except .usage.ignoreusers; }; GetUsersHDB:{[date] - handle:hopen hsym `$raze"::",string (first -1?exec port from .servers.procstab where proctype=`queryhdb),":querygateway:pass"; - usageusers:handle"first flip select distinct u from usage where date=",string date; - ignoreusers:`,(`$system"echo $USER"),`admin,exec distinct proctype from .servers.procstab; - res:usageusers except ignoreusers; - if[1=count res; :first res]; - :res; + query:"select cmd from usage where date=", (.Q.s1 date), ", u=`gateway"; + handle:GetHandle `queryrdb; + res:raze last .async.deferred[handle; query]; + resparsed:ParseCmd res; + + users:first value flip select distinct originaluser from resparsed; + :realusers:users except .usage.ignoreusers; }; GetHandle:{[proc] From 788c189dafd319cd001c69f83a5c5083da65e96c Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 14:00:46 +0000 Subject: [PATCH 105/114] Add discovery and stp to .usage.ignoreusers --- config/settings/default.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings/default.q b/config/settings/default.q index fb0e8341b..6af8b937f 100644 --- a/config/settings/default.q +++ b/config/settings/default.q @@ -21,7 +21,7 @@ logtomemory:1b // write query logs to memory ignore:1b // check the ignore list for functions to ignore ignorelist:(`upd;"upd") // the list of functions to ignore in async calls allowedusers:`angus`michael`stephen -ignoreusers:raze (`admin`rdb`reporter`sort`wdb`monitor`queryrdb`queryfeed`querygateway`gateway`dqc`dqe;.z.u;`) // clients to ignore for query logging +ignoreusers:raze (`discovery`dqc`dqe`gateway`mburns`monitor`queryfeed`querygateway`queryrdb`rdb`reporter`segmentedtickerplant`sort`wdb;.z.u;`) // clients to ignore for query logging flushinterval:0D00:30:00 // default value for how often to flush the in-memory logs flushtime:1D00 // default value for how long to persist the in-memory logs. Set to 0D for no flushing suppressalias:0b // whether to suppress the log file alias creation From 03f1efa4580af2f57a2aa98dbae3371efdf9ebd0 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 15:44:15 +0000 Subject: [PATCH 106/114] Fix get user fn --- code/querygateway/querygatewayfuncs.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index d16aa7b8e..489c03d11 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -125,7 +125,7 @@ GetUsersRDB:{ GetUsersHDB:{[date] query:"select cmd from usage where date=", (.Q.s1 date), ", u=`gateway"; - handle:GetHandle `queryrdb; + handle:GetHandle `queryhdb; res:raze last .async.deferred[handle; query]; resparsed:ParseCmd res; From c20bf0babb58f174779c30da7333d48084214cbe Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 15:47:49 +0000 Subject: [PATCH 107/114] Remove old GetUsers fn --- code/querygateway/querygatewayfuncs.q | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 489c03d11..511dd65d5 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -103,16 +103,6 @@ GetDateRange:{[query] //HistoricalDates:{}; - - -GetUsers:{ - query:"first value flip select distinct u from .clients.clients where not u in .usage.ignoreusers"; - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`gateway; - res:handle query; - if[1=count res; :first res]; - :res; - }; - GetUsersRDB:{ query:"select cmd from usage where u=`gateway"; handle:GetHandle `queryrdb; From a536b6e51b269c28736337d9c7c4d06a47232576 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 16:41:09 +0000 Subject: [PATCH 108/114] Fix error trap --- code/querygateway/querygatewayfuncs.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index 511dd65d5..b765eafa0 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -130,8 +130,8 @@ GetHandle:{[proc] // currently setup to deal with ubiquitous error sting in cmd // will need updated when the foregoing is fixed ParseCmd:{[res] - cmdsplit:@[{select cmd:-2#'";" vs/: cmd from x}; res; {.lg.e[`ParseCmd; "cmd col missing from paramater res, returning input and exiting function"]; :res;}]; - remainder:@[{select from (cols[res] except `cmd)#x}; res; ()]; + cmdsplit:@[{select cmd:-2#'";" vs/: cmd from x}; res; {.lg.e[`ParseCmd; "cmd col missing from parameter res, returning input and exiting function"]; :res;}]; + remainder:@[(cols[res] except `cmd)#; res; ()]; cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; From fb141a73ed7d3ef94031ea27adb0658d7d4ec052 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Tue, 7 Mar 2023 18:40:43 +0000 Subject: [PATCH 109/114] Start adding user picker logic for realtime fns --- code/querygateway/querygatewayfuncs.q | 33 ++++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q index b765eafa0..1c32e417d 100644 --- a/code/querygateway/querygatewayfuncs.q +++ b/code/querygateway/querygatewayfuncs.q @@ -157,25 +157,35 @@ ProcPickerHDB:{[process] :phrase; }; -QueryCountsRealtime:{[process] - users:GetUsersRDB[]; +UserParseRDB:{[user] + $[user=`all; + phrase:GetUsersRDB[]; + phrase:enlist user]; + + :phrase; + }; + +QueryCountsRealtime:{[process; user] procphrase:ProcPickerRDB[`$process]; query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; - - :select count i from ParseCmd[res] where originaluser in users; + + userparsed:UserParseRDB `$user; + + :select count i from ParseCmd[res] where originaluser in userparsed; }; QueryUserCountsRealtime:{[process] - users:GetUsersRDB[]; procphrase:ProcPickerRDB[`$process]; query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; + users:GetUsersRDB[]; + :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; }; @@ -211,21 +221,22 @@ QueryUserCountsHistorical:{[date; process] :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; }; -PeakUsage:{[process] - users:GetUsersRDB[]; +PeakUsage:{[process; user] procphrase:ProcPickerRDB[`$process]; query:"select time, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; handle:GetHandle `queryrdb; res:raze last .async.deferred[handle; query]; - resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; + userparsed:UserParseRDB `$user; + + resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in userparsed; // select separate tables of times and queries for each user - getquerycounts:{[resparsed; users] ?[resparsed; enlist(in; `originaluser; `users); 0b; (`time`queries)!(`time`queries)]}[resparsed; ]; - querycounts:getquerycounts'[users]; + getquerycounts:{[resparsed; userparsed] ?[resparsed; enlist(in; `originaluser; `userparsed); 0b; (`time`queries)!(`time`queries)]}[resparsed; ]; + querycounts:getquerycounts'[userparsed]; // rename 'queries' col with name of user for each table - querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; + querycountsn:{:(`time; y) xcol x;}'[querycounts; userparsed]; peakusage:0!(pj/)1!'querycountsn; From d8618fb863f9c6c0644d75632d35d942d2745ce9 Mon Sep 17 00:00:00 2001 From: Michael Burns Date: Wed, 22 Mar 2023 10:49:54 +0000 Subject: [PATCH 110/114] Add data normalisation to queryfeed --- code/processes/queryfeed.q | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/processes/queryfeed.q b/code/processes/queryfeed.q index e20b99a52..523aadeaf 100644 --- a/code/processes/queryfeed.q +++ b/code/processes/queryfeed.q @@ -28,7 +28,11 @@ readlog:{[file] @[{update "J"$'" " vs' mem from flip (cols `us)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]}; queryfeed:{ - h(".u.upd";`usage;value flip select from `us); + // normalise cmd data for gateway users + usnorm:update cmd:-2#'";" vs' cmd from us where user=`gateway; + usnorm:update cmd:first each cmd from usnorm where (first each cmd)~'(last each cmd); + + h(".u.upd";`usage;value flip select from usnorm); us::0#us; }; From ad3283c1424e28265dd9e82a58a0045478359a2c Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Wed, 24 May 2023 17:03:50 +0100 Subject: [PATCH 111/114] remove querygatewayfuncs.q --- code/querygateway/querygatewayfuncs.q | 394 -------------------------- 1 file changed, 394 deletions(-) delete mode 100644 code/querygateway/querygatewayfuncs.q diff --git a/code/querygateway/querygatewayfuncs.q b/code/querygateway/querygatewayfuncs.q deleted file mode 100644 index 1c32e417d..000000000 --- a/code/querygateway/querygatewayfuncs.q +++ /dev/null @@ -1,394 +0,0 @@ -//Number of queries on a given process by hour (or any given bucket) -testfunc:{.gw.syncexec[({[x;y]select query_suc:count i by procname from usage where time within (x;y)};x;y);`queryrdb]} -//Number of queries ran by a specific user -//Number of distinct users querying a process -//Return list of queries that run over a given run time -//Return the date range queried on a process - - -/Number of queries on a given process by hour (or any given bucket) -//d:(`proc`bucket`sd`ed)!(`rdb1;60:.z.d-3;.z.d) -ProcSucErr:{[d] - .gw.syncexec[({[d] - $[.proc.proctype=`queryhdb; - select queries_suc:count where status=first string `c,queries_err:count where status=first string `e - by time.date,d[`bucket] xbar time.minute from usage - where date within (d[`sd];d[`ed]),procname=d[`proc]; - select queries_suc:count where status=first string `c,queries_err:count where status=first string `e - by time.date,d[`bucket] xbar time.minute from usage - where procname=d[`proc]]};d);`queryhdb`queryrdb]; - }; -//ProcSucErrFF:{[`proc;`bucket;sd;ed] -// .gw.syncexec[({[`proc;`bucket;sd;ed] -// $[.proc.proctype=`queryhdb; -// -// ?[`usage;((within;`date;(enlist;`sd;`ed));(=;`procname;`proc)); -// `date`minute!(`time.date;(k){x*y div x:$[16h=abs[@x];"j"$x;x]};`bucket;`time.minute); -// `queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]; -// -// ?[`usage;enlist((=;`procname;`proc)); -// `date`minute!(`time.date;(k){x*y div x:$[16h=abs[@x];"j"$x;x]};`bucket;`time.minute); -// `queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]]};`proc;`bucket;sd;ed);`queryhdb`queryrdb]; -// }; -//Number of queries ran by a specific user -//d:(`user`sd`ed)!(`acreehay;.z.d-3;.z.d) -QueryByUser:{[d] - .gw.syncexec[({[d] - $[.proc.proctype=`queryhdb; - select queries_suc:count where status=first string `c,queries_err:count where status=first string `e - by time.date,procname from usage - where date within (d[`sd];d[`ed]),user=d[`user]; - select queries_suc:count where status=first string `c,queries_err:count where status=first string `e - by time.date,procname from usage - where user=d[`user]]};d);`queryhdb`queryrdb]; - }; - -//QuerByUser function in functional form -//Same arguments as above just not in dictionary form -QueryByUserFF:{[username;sd;ed] - .gw.syncexec[({[username;sd;ed] - $[.proc.proctype=`queryhdb; - ?[`usage;((within;`date;(enlist;`sd;`ed));(=;`user;`username));`date`procname!`time.date`procname;`queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]; - ?[`usage;enlist((=;`user;`username));`date`procname!`time.date`procname;`queries_suc`queries_err!((#:;(&:;(=;`status;(*:;($:;enlist(`c))))));(#:;(&:;(=;`status;(*:;($:;enlist(`e)))))))]]};username;sd;ed);`queryhdb`queryrdb]; - }; - - -//Number of distinct users querying a process -//d:(`proc`bucket`sd`ed)!(`rdb1;60;.z.d-3;.z.d) -DistinctUsers:{[d] - .gw.syncexec[({[d] - $[.proc.proctype=`queryhdb; - select unique_users:count distinct user, users:distinct user - by time.date,d[`bucket] xbar time.minute from usage - where date within (d[`sd];d[`ed]),procname=d[`proc]; - select unique_users:count distinct user, users:distinct user - by time.date,d[`bucket] xbar time.minute from usage - where procname=d[`proc]]};d);`queryhdb`queryrdb]; - }; -//Return list of queries that run over a given run time -//10000;.z.d-3;.z.d -RuntimeLimit:{[lim;sd;ed] - .gw.syncexec[({[lim;sd;ed] - $[.proc.proctype=`queryhdb; - select from usage - where date within (sd;ed),runtime>lim; - select from usage - where runtime>lim]};lim;sd;ed);`queryhdb`queryrdb]; - }; - -//RuntimeLimit function in functional form -RuntimeLimitFF:{[lim;sd;ed] - .gw.syncexec[({[lim;sd;ed] - $[.proc.proctype=`queryhdb; - ?[`usage;((within;`date;(enlist;`sd;`ed));(>;`runtime;`lim));0b;()]; - ?[`usage;enlist((>;`runtime;`lim));0b;()]]};lim;sd;ed);`queryhdb`queryrdb]; - }; - -GetDateRange:{[query] - wherephrase:raze raze (parse query) 2; - dateindex:1 + wherephrase?`date; - // index into wherephrase to retrieve date - date:wherephrase dateindex; - - // if date is a range, drop the 'enlist' element - if[3=count date; date:1_date]; - - :eval each date; - }; - -//functions to set up variables -//RealtimeProcs:{}; - -//HistoricalProcs{}; - -//HistoricalDates:{}; - -GetUsersRDB:{ - query:"select cmd from usage where u=`gateway"; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd res; - - users:first value flip select distinct originaluser from resparsed; - :realusers:users except .usage.ignoreusers; - }; - -GetUsersHDB:{[date] - query:"select cmd from usage where date=", (.Q.s1 date), ", u=`gateway"; - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd res; - - users:first value flip select distinct originaluser from resparsed; - :realusers:users except .usage.ignoreusers; - }; - -GetHandle:{[proc] - :first -1?exec handle from .gw.availableserverstable[1b] where servertype=proc; - }; - -// currently setup to deal with ubiquitous error sting in cmd -// will need updated when the foregoing is fixed -ParseCmd:{[res] - cmdsplit:@[{select cmd:-2#'";" vs/: cmd from x}; res; {.lg.e[`ParseCmd; "cmd col missing from parameter res, returning input and exiting function"]; :res;}]; - remainder:@[(cols[res] except `cmd)#; res; ()]; - - cmdcolsplit:select originaluser, query from @[cmdsplit; `originaluser`query; :; flip cmdsplit`cmd]; - cmdcolsplitparsed:update originaluser:`$1_'originaluser, query:1_'-3_'query from cmdcolsplit; - - if [0=count remainder; :cmdcolsplitparsed;]; - - :remainder,'cmdcolsplitparsed; - }; - -ProcPickerRDB:{[process] - $[process=`all; - phrase:"proctype=`rdb"; - phrase:"procname=", .Q.s1 process]; - - :phrase; - }; - -ProcPickerHDB:{[process] - $[process=`all; - phrase:"proctype=`hdb"; - phrase:"procname=", .Q.s1 process]; - - :phrase; - }; - -UserParseRDB:{[user] - $[user=`all; - phrase:GetUsersRDB[]; - phrase:enlist user]; - - :phrase; - }; - -QueryCountsRealtime:{[process; user] - procphrase:ProcPickerRDB[`$process]; - - query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - - userparsed:UserParseRDB `$user; - - :select count i from ParseCmd[res] where originaluser in userparsed; - }; - -QueryUserCountsRealtime:{[process] - procphrase:ProcPickerRDB[`$process]; - - query:"select from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - - users:GetUsersRDB[]; - - :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; - }; - -QueryCountsHistorical:{[date; process] - users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process]; - - $[.z.d<=date; query:(); // log error - 1=count date; query:"select from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - 2=count date; query:"select queries:count i from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), "),", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - // log error - query:()] - - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; raze query]; - - :select queries:count i from ParseCmd[res] where originaluser in users; - }; - -QueryUserCountsHistorical:{[date; process] - users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process]; - - $[.z.d<=date; query:(); // log error - 1=count date; query:"select from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - 2=count date; query:"select from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - // log error - query:()] - - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; raze query]; - - :select queries:count i by originaluser from ParseCmd[res] where originaluser in users; - }; - -PeakUsage:{[process; user] - procphrase:ProcPickerRDB[`$process]; - - query:"select time, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - - userparsed:UserParseRDB `$user; - - resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in userparsed; - - // select separate tables of times and queries for each user - getquerycounts:{[resparsed; userparsed] ?[resparsed; enlist(in; `originaluser; `userparsed); 0b; (`time`queries)!(`time`queries)]}[resparsed; ]; - querycounts:getquerycounts'[userparsed]; - // rename 'queries' col with name of user for each table - querycountsn:{:(`time; y) xcol x;}'[querycounts; userparsed]; - - peakusage:0!(pj/)1!'querycountsn; - - :update time:.z.d + time from peakusage; - }; - -//LongestRunning:{[process] -// users:GetUsersRDB[]; -// query:"select time, runtime, u, cmd from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; -// handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; -// res:raze last .async.deferred[handle; query]; -// :ParseCmd res; -// }; - -LongestRunningHeatMap:{[process] - users:GetUsersRDB[]; - procphrase:ProcPickerRDB[`$process]; - - query:"select time, runtime, proctype, procname, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd[res]; - - :select time:.z.d + 10 xbar time.minute, runtime, proctype, procname, originaluser, query from resparsed where originaluser in users, runtime=(max; runtime) fby 10 xbar time.minute; - }; - -//Return percentage of queries that were successful by user -//QueryErrorPercentage:{[process] -// users:GetUsersRDB[]; -// query:"select completed:100*(count i where status=\"c\")%(count i where status=\"c\")+count i where status=\"e\" by u from usage where procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users); -// handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; -// res:raze last .async.deferred[handle; query]; -// :res; -// }; - -QueryErrorPercentage:{[process] - users:GetUsersRDB[]; - procphrase:ProcPickerRDB[`$process]; - - // where status is "c" or "e" - query:"select status, cmd from usage where u=`gateway, status in ", (string `ce), ", ", procphrase; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - - resparsed:ParseCmd[res]; - - :select completed:count i where status="c", error:count i where status="e" from resparsed where originaluser in users; - }; - -LongestRunning:{[process] - users:GetUsersRDB[]; - procphrase:ProcPickerRDB[`$process]; - - query:"select runtime, cmd from usage where u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryrdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd[res]; - - :select max runtime by originaluser from resparsed where originaluser in users; - }; - -LongestRunningHistorical:{[date; process] - users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process]; - - $[.z.d<=date; query:(); // log error - 1=count date; query:"select runtime, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - 2=count date; query:"select runtime, cmd from usage where date within (", (.Q.s1 first date), "; ", (.Q.s1 last date), ")", ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - // log error - query:()] - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd[res]; - - :select max runtime by originaluser from resparsed where originaluser in users; - }; - -QueryErrorPercentageHistorical:{[date; process] - users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process]; - - $[.z.d<=date; query:(); // log error - // where status is "c" or "e" - 1=count date; query:"select status, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status in ", (string `ce), ", ", procphrase; - // where status is "c" or "e" - 2=count date; query:"select status, cmd from usage where date in (", (.Q.s1 first date), "; ", (.Q.s1 last date), "), u=`gateway, status in ", (string `ce), ", ", procphrase; - // log error - query:()] - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd:[res]; - - :select completed:count i where status="c", error:count i where status="e" from resparsed where originaluser in users; - }; - -//Return queries which take longer than given runtime input, t -//t in milliseconds 10^-3 -LongQuery:{[t] - users:GetUsers[]; - timens:t*1000000; //Convert to nano seconds - query:"select from usage where runtime>", (.Q.s1 timens), ", u in ", (.Q.s1 users); - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:raze last .async.deferred[handle; query]; - :res; - }; - -//Number of distinct users -NumberOfUsers:{ - users:GetUsers[]; - query:"select count distinct u from usage where u in ", (.Q.s1 users); - handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryrdb; - res:raze last .async.deferred[handle; query]; - :res; - }; - -PeakUsageHistorical:{[date; process] - users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process]; - - query:"select time, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; query]; - - resparsed:`time xcol 0!select queries:count i by 10 xbar time.minute, originaluser from ParseCmd[res] where originaluser in users; - - // select separate tables of times and queries for each user - getquerycounts:{[resparsed; users] ?[resparsed; enlist(in; `originaluser; `users); 0b; (`time`queries)!(`time`queries)]}[resparsed; ]; - querycounts:getquerycounts'[users]; - // rename 'queries' col with name of user for each table - querycountsn:{:(`time; y) xcol x;}'[querycounts; users]; - - peakusage:0!(pj/)1!'querycountsn; - - :update time:.z.d + time from peakusage; - }; - -//LongestRunningHistorical:{[date;process] -// users:GetUsersHDB[date]; -// query:"select time, runtime, u, cmd from usage where date=", (.Q.s1 date), ", procname in ", (.Q.s1 process), ", u in ", (.Q.s1 users), ", runtime=max runtime"; -// handle:first -1?exec handle from .gw.availableserverstable[1b] where servertype=`queryhdb; -// res:raze last .async.deferred[handle; query]; -// :ParseCmd res; -// }; - -LongestRunningHeatMapHistorical:{[date; process] - users:GetUsersHDB[date]; - procphrase:ProcPickerHDB[`$process]; - - query:"select time, runtime, proctype, procname, cmd from usage where date=", (.Q.s1 date), ", u=`gateway, status=", (.Q.s1 "c"), ", ", procphrase; - handle:GetHandle `queryhdb; - res:raze last .async.deferred[handle; query]; - resparsed:ParseCmd[res]; - - :select time:.z.d + 10 xbar time.minute, runtime, proctype, procname, originaluser, query from resparsed where originaluser in users, runtime=(max; runtime) fby 10 xbar time.minute; - }; From 1b4a2a0a37f58231e6e87c6917d58dd0b0c57d7c Mon Sep 17 00:00:00 2001 From: Matthew Deshpande Date: Thu, 25 May 2023 12:17:01 +0100 Subject: [PATCH 112/114] Updating QueryManagement.md with PR comment changes --- docs/QueryManagement.md | 42 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/docs/QueryManagement.md b/docs/QueryManagement.md index ad4982a10..b241bcac0 100644 --- a/docs/QueryManagement.md +++ b/docs/QueryManagement.md @@ -1,9 +1,8 @@ -Functionality Overview -====================== +## Functionality Overview -Query Logging Management is an addition to TorQ to enhance the current query logging system. The aim for this tool is to have access to information about queries that are sent to specific processes throughout the day, allowing for query analysis to be carried. +Query Logging Management is an addition to TorQ to enhance the current query logging system. The aim for this tool is to have access to information about queries that are sent to specific processes throughout the day, allowing for query analysis to be carried out. For each query executed we want access to: @@ -18,45 +17,37 @@ For each query executed we want access to: This query logging functionality can be enabled or disabled for specific processes using variables within TorQ config files. -Architecture -============ +## Architecture The architecture of the Query Logging Management framework is shown in the following diagram: ![QueryLoggingManagementArchitecture](graphics/torq-qlm_architecture.PNG) -Processes -======== +## Processes -Query Feed ----------- +#### Query Feed -If enabled within a TorQ process, the Query Feed process subscribes to updates from the .usage.usage table that is defined in the logusage.q handlers script from all enabled TorQ processes. +If enabled within a TorQ stack, the Query Feed process subscribes to updates from the .usage.usage table that is defined in the logusage.q handlers script from all enabled TorQ processes. -Once a connection has been set up to our Query Tickerplant, the Query feed sends a message to execute the .u.upd function to inesrt this collected .usage.usage table into a usage table. +Once a connection has been set up to our Query Tickerplant, the Query feed sends a message to execute the .u.upd function to insert this collected .usage.usage table into a usage table. -Query Tickerplant ------------------ +#### Query Tickerplant The Query Tickerplant process receives updates from the Query Feed process regarding the usage table and publishes it on to any subscribing process, operating the same way as a standard tickerplant. -Query RDB ---------- +#### Query RDB The Query RDB works like a normal RDB process, receiving usage table messages from the Tickerplant and storing this in memory in order to be queried. At the end of day, the usage table is saved down onto disk to be loaded into the HDB. -Query HDB ---------- +#### Query HDB The Query HDB loads in historical usage data from disk in order for long term query information to be queried. -Query Gateway -------------- +#### Query Gateway -The Query Gateway process subscribes to our specific Query RDB and HDB processes to load balance queries and allow for access to query information involving both historical and real time data. The query gateway process also contains a number of analytics functions to return specific aggregations from the usage table (i.e how many queries were executed for each process on a specific day). +The Query Gateway process connects to our specific Query RDB and HDB processes to load balance queries and allow for access to query information involving both historical and real time data. The query gateway process also contains a number of analytics functions to return specific aggregations from the usage table (e.g. how many queries were executed for each process on a specific day). -Table Contents -============== +## Table Contents The Query usage table contains the follow columns: @@ -82,12 +73,11 @@ The Query usage table contains the follow columns: - mem: Memory information connected to the query execution -- sz: The byte size of the result from the query being executed +- sz: The size in bytes of the result from the query being executed -- error: An error string +- error: An error string, if error occured (i.e. status is `e`) -Setup -===== +## Setup The TorQ Query Management logging tool can be added on top of any existing TorQ setup or the full TorQ application including Query logging can be integrated with data sources and feeds. From 4a871267c0432982253494c510cf854c2abf6ced Mon Sep 17 00:00:00 2001 From: StephenBoyd1 Date: Tue, 30 May 2023 11:41:30 +0100 Subject: [PATCH 113/114] update logusage.q with PR changes --- code/handlers/logusage.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index 98f69f106..4ac303b28 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -10,7 +10,7 @@ \d .usage // table to store usage info -usage:@[value;`usage;([]time:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$(); procname:`symbol$(); status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())] +usage:@[value;`usage;([]time:`timestamp$();id:`long$();timer:`long$();zcmd:`symbol$();proctype:`symbol$(); procname:`symbol$(); status:`char$();a:`int$();u:`symbol$();w:`int$();cmd:();mem:();sz:`long$();error:())] // Check if the process has been initialised correctly if[not @[value;`.proc.loaded;0b]; '"environment is not initialised correctly to load this script"] @@ -77,7 +77,7 @@ readlog:{[file] update zcmd:`$1 _' string zcmd, procname:`$1 _' string procname, proctype:`$1 _' string proctype, u:`$1 _' string u, a:"I"$-1 _' a, w:"I"$-1 _' w from // Read in file - @[{update "J"$'" " vs' mem from flip (cols .usage.usage)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]} + @[{update "J"$'" " vs' mem from flip (cols .usage.usage)!("PJJSSSC*S***JS";"|")0:x};hsym`$file;{'"failed to read log file : ",x}]} // roll the logs // inmemorypersist = the number From dce824deaa3f7b2d8cdde29bef74c4e81bae477c Mon Sep 17 00:00:00 2001 From: Matthew Deshpande Date: Tue, 30 May 2023 14:15:09 +0100 Subject: [PATCH 114/114] changing querytrack from csv to txt file type --- code/handlers/logusage.q | 2 +- config/{querytrack.csv => querytrack.txt} | 0 config/settings/queryfeed.q | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename config/{querytrack.csv => querytrack.txt} (100%) diff --git a/code/handlers/logusage.q b/code/handlers/logusage.q index 98f69f106..45220659f 100644 --- a/code/handlers/logusage.q +++ b/code/handlers/logusage.q @@ -32,7 +32,7 @@ LEVEL:@[value;`LEVEL;3] // Log level querytrack:@[value;`querytrack;0b] // whether query tracking is enabled by default // enable query tracking for proc if procname included in csv config file -querytrack:$[.proc.procname in "S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.csv";1b;0b] +querytrack:$[.proc.procname in "S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.txt";1b;0b] id:@[value;`id;0j] nextid:{:id+::1} diff --git a/config/querytrack.csv b/config/querytrack.txt similarity index 100% rename from config/querytrack.csv rename to config/querytrack.txt diff --git a/config/settings/queryfeed.q b/config/settings/queryfeed.q index 1fa5c552b..58ca14222 100644 --- a/config/settings/queryfeed.q +++ b/config/settings/queryfeed.q @@ -4,7 +4,7 @@ CONNECTIONS:`querytp; // Feedhandler connects to the query-tickerplant HOPENTIMEOUT:30000 \d . -subprocs:"S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.csv"; // List of procs for query-tickerplant to subscribe to +subprocs:"S"$read0 hsym `$(getenv `KDBCONFIG),"/querytrack.txt"; // List of procs for query-tickerplant to subscribe to reloadenabled:1b \d .usage