From a4131a83fb278eff069eac842cdb16a831cd0b94 Mon Sep 17 00:00:00 2001 From: jballantine Date: Fri, 28 Mar 2025 17:22:02 +0000 Subject: [PATCH 1/3] Make idb work with dataaccess lib --- code/common/checkinputs.q | 2 ++ code/dataaccess/dataaccessutils.q | 2 +- code/gateway/dataaccess.q | 10 +++++++--- code/processes/idb.q | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/code/common/checkinputs.q b/code/common/checkinputs.q index 4dae11ac7..0c73d207d 100755 --- a/code/common/checkinputs.q +++ b/code/common/checkinputs.q @@ -226,3 +226,5 @@ checkpostback:{[dict;parameter] checktimeout:{[dict;parameter] checktype[-16h;dict;parameter]; :dict}; + +checkprocs:{[dict;parameter]:.checkinputs.checktype[-11 11h;dict;parameter];}; diff --git a/code/dataaccess/dataaccessutils.q b/code/dataaccess/dataaccessutils.q index fa90de91f..e9a536431 100644 --- a/code/dataaccess/dataaccessutils.q +++ b/code/dataaccess/dataaccessutils.q @@ -5,7 +5,7 @@ readtableproperties:{[tablepropertiepath] .lg.o[`readtableproperties;"loading table properties"]; table:`tablename`proctype xkey readcsv[tablepropertiepath;"ssssstsss"]; //read in table from file alltable:?[table;enlist(in;`proctype;enlist`all`);0b;()]; //find any instance of the use "all" or blank for proctype - table:table,![alltable;();0b;(enlist`proctype)!enlist(enlist `hdb)],![alltable;();0b;(enlist`proctype)!enlist(enlist `rdb)]; //join rdb and hdb entries for any "all" or blank entries + table:table,raze{![x;();0b;(enlist`proctype)!enlist(enlist y)]}[alltable]each`rdb`hdb`idb; //join rdb, idb and hdb entries for any "all" or blank entries table:![table;enlist(in;`proctype;enlist`all`);0b;`symbol$()]; //remove "all" or blank entries from table table:?[table;$[.proc.proctype=`gateway;();enlist(=;`proctype;`.proc.proctype)];0b;()]; table:update .eodtime.datatimezone ^ datatimezone, .eodtime.rolltimeoffset ^ rolltimeoffset,.eodtime.rolltimezone^rolltimezone from table; diff --git a/code/gateway/dataaccess.q b/code/gateway/dataaccess.q index 391569dba..773e809bc 100644 --- a/code/gateway/dataaccess.q +++ b/code/gateway/dataaccess.q @@ -154,6 +154,8 @@ partdict:{[input] tabname:input[`tablename]; // Remove duplicate servertypes from the gw.servers servers:select from .gw.servers where i=(first;i)fby servertype; + // Only target specified procs if defined + if[`procs in key input;servers:select from servers where servertype in ((),input`procs)]; // extract the procs which have the table defined servers:select from servers where {[x;tabname]tabname in @[x;`tables]}[;tabname] each attributes; // Create a dictionary of the attributes against servertypes @@ -162,6 +164,8 @@ partdict:{[input] procdict:@[procdict;key procdict;{[x;tabname]if[99h=type x;:x[tabname]];:x}[;tabname]]; // returns the dictionary as min date/ max date procdict:asc @[procdict;key procdict;{:(min x; max x)}]; + // Let idb take precedence over rdb to prevent data duplication + if[all `rdb`idb in key procdict;procdict:delete rdb from procdict]; // prevents overlap if more than one process contains a specified date if[1 Date: Mon, 31 Mar 2025 13:44:32 +0100 Subject: [PATCH 2/3] Make idb work with dataaccess lib --- code/gateway/dataaccess.q | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/code/gateway/dataaccess.q b/code/gateway/dataaccess.q index 773e809bc..3757dc97d 100644 --- a/code/gateway/dataaccess.q +++ b/code/gateway/dataaccess.q @@ -197,16 +197,11 @@ adjustqueries:{[options;part] partitions:possparts{(min x;max x)}'[partitions]; partitions:`timestamp$partitions; - // adjust the times to account for period end time when int partitioned - partitions:{x[0],-1+x[1]+1D}each partitions; - - // if start/end time not a date, then adjust dates parameter for the correct types - if[not a:-12h~tp:type start:options`starttime; - // converts partitions dictionary to timestamps/datetimes - partitions:$[-15h~tp;"z"$;]{(0D+x 0;x[1]+1D-1)}'[partitions]; - // convert first and last timestamp to start and end time - partitions:@[partitions;f;:;(start;partitions[f:first key partitions;1])]; - partitions:@[partitions;l;:;(partitions[l:last key partitions;0];options`endtime)]]; + // adjust the query times accordingly + options:@[@[options;`starttime;:;"p"$options`starttime];`endtime;:;$[-14h~type et:options`endtime;-1+1D+et;"p"$et]]; //ensure st/et are timestamps; if date adjust endtime + partitions:{x[0],x[1]+1D-1}each partitions; //create dict of datetime coverage for each process + partitions:@[partitions;f;:;(st:"p"$options`starttime;min(et:"p"$options`endtime;partitions[f:first key partitions;1]))]; //amend query datetimes on hdb + partitions:@[partitions;l;:;(max(st;partitions[l:last key partitions;0]);et)]; //amend query datetimes on rdb/idb // adjust map reducable aggregations to get correct components if[(1 Date: Fri, 4 Apr 2025 09:28:02 +0100 Subject: [PATCH 3/3] changes following JMM's feedback --- code/gateway/dataaccess.q | 40 ++++++++++----------------------------- code/processes/idb.q | 2 +- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/code/gateway/dataaccess.q b/code/gateway/dataaccess.q index 3757dc97d..434b25a98 100644 --- a/code/gateway/dataaccess.q +++ b/code/gateway/dataaccess.q @@ -164,11 +164,6 @@ partdict:{[input] procdict:@[procdict;key procdict;{[x;tabname]if[99h=type x;:x[tabname]];:x}[;tabname]]; // returns the dictionary as min date/ max date procdict:asc @[procdict;key procdict;{:(min x; max x)}]; - // Let idb take precedence over rdb to prevent data duplication - if[all `rdb`idb in key procdict;procdict:delete rdb from procdict]; - // prevents overlap if more than one process contains a specified date - if[1count p:options`procs;:options]; - // get the tablename - tabname:options[`tablename]; - // remove duplicate servertypes from the gw.servers - servers:select from .gw.servers where i=(first;i)fby servertype; - // only target specified procs if defined - if[`procs in key options;servers:select from servers where servertype in ((),options`procs)]; - // extract the procs which have the table defined - servers:select from servers where {[x;tabname]tabname in @[x;`tables]}[;tabname] each attributes; - // create a dictionary of the attributes against servertypes - procdict:exec servertype!attributes[;`partition] from servers; - // if the response is a dictionary index into the tablename - procdict:@[procdict;key procdict;{[x;tabname]if[99h=type x;:x[tabname]];:x}[;tabname]]; - // create list of all available partitions - possparts:raze value procdict; - - //group partitions to relevant process - partitions:group key[part]where each{within[y;]each value x}[part]'[possparts]; - partitions:possparts{(min x;max x)}'[partitions]; - partitions:`timestamp$partitions; - - // adjust the query times accordingly - options:@[@[options;`starttime;:;"p"$options`starttime];`endtime;:;$[-14h~type et:options`endtime;-1+1D+et;"p"$et]]; //ensure st/et are timestamps; if date adjust endtime - partitions:{x[0],x[1]+1D-1}each partitions; //create dict of datetime coverage for each process - partitions:@[partitions;f;:;(st:"p"$options`starttime;min(et:"p"$options`endtime;partitions[f:first key partitions;1]))]; //amend query datetimes on hdb - partitions:@[partitions;l;:;(max(st;partitions[l:last key partitions;0]);et)]; //amend query datetimes on rdb/idb + + // ensure st/et are timestamps; if date adjust endtime + options:@[@[options;`starttime;:;st:"p"$options`starttime];`endtime;:;et:$[-14h~type et:options`endtime;-1+1D+et;"p"$et]]; + // create dict of datetime coverage for each process + partitions:{"p"$x[0],x[1]+1D-1}each part; + // amend query datetimes on hdb + if[`hdb in key partitions;partitions:@[partitions;`hdb;:;(st;min(et;partitions[`hdb;1]))]]; + // amend query datetimes on rdb/idb + if[`idb in key partitions;partitions:@[partitions;`idb;:;(max(st;partitions[`idb;0]);et)]]; + if[`rdb in key partitions;partitions:@[partitions;`rdb;:;(max(st;partitions[`rdb;0]);et)]]; // adjust map reducable aggregations to get correct components if[(1