diff --git a/nifi/user-templates/CogStack_Cohort_create_source_docs.xml b/nifi/user-templates/CogStack_Cohort_create_source_docs.xml
index 03d4b2b9..59cc540f 100644
--- a/nifi/user-templates/CogStack_Cohort_create_source_docs.xml
+++ b/nifi/user-templates/CogStack_Cohort_create_source_docs.xml
@@ -390,6 +390,34 @@
ba40cb68-0d02-3fdf-acf7-6be169ebdf91
Exports annotations and records that are used by CogStack-Cohort.
+
+ 01430d51-2cd4-3c5f-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 01430d51-2cd4-3c5f-9c62-298aeeeff916
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 3fd696b4-944f-3364-0000-000000000000
+ FUNNEL
+ 3fd696b4-944f-3364-b0a8-09968605fe97
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ failure
+
+ 0
+
0191cd63-24f0-337b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -503,6 +531,34 @@
0
+
+ 13047b3d-4536-3552-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 13047b3d-4536-3552-a609-543763b73733
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ bfe47c8f-331e-39a4-0000-000000000000
+ PROCESSOR
+ bfe47c8f-331e-39a4-b23e-1a7eb2373e40
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ 0
+
15105083-2b5c-3325-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -671,6 +727,90 @@
0
+
+ 2cd2dd0a-1a77-3db2-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 2cd2dd0a-1a77-3db2-94a4-f0e6f37b4dea
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 440d39c2-9205-3aaf-0000-000000000000
+ PROCESSOR
+ 440d39c2-9205-3aaf-8def-38cc36b5a733
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ 0
+
+
+ 41694b54-995a-3769-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 41694b54-995a-3769-970c-12eb6d4e5f63
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 8990354d-6606-3294-0000-000000000000
+ PROCESSOR
+ 8990354d-6606-3294-a0b5-64036dd8a8dd
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ merged
+
+ 0
+
+
+ 45f295a3-e66a-3bbf-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 45f295a3-e66a-3bbf-b01e-7bea92709975
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 2c150f1f-1273-3eab-0000-000000000000
+ PROCESSOR
+ 2c150f1f-1273-3eab-91f0-4e39828ec66d
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ 0
+
4eeb5b2f-8b38-3229-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -755,6 +895,34 @@
0
+
+ 5a2100a5-1ca9-3cbc-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 5a2100a5-1ca9-3cbc-915d-00df44e37057
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 7dd60db1-2d31-351c-0000-000000000000
+ PROCESSOR
+ 7dd60db1-2d31-351c-89c1-c2ed37eb4567
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ 0
+
5f12c6b3-cc4d-3157-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -783,6 +951,34 @@
0
+
+ 5f8703f3-3856-3e21-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 5f8703f3-3856-3e21-9e06-1a119e252f1d
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 59dda431-fb7c-3a65-0000-000000000000
+ PROCESSOR
+ 59dda431-fb7c-3a65-b2d7-1b0980c0b915
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ failure
+
+ 0
+
605ad7df-4119-3ed7-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -867,33 +1063,6 @@
0
-
- 79364360-9ad9-3049-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
- 79364360-9ad9-3049-b574-219466bf2f3f
- 1 GB
- 10000
-
- ba40cb68-0d02-3fdf-0000-000000000000
- bfe47c8f-331e-39a4-0000-000000000000
- PROCESSOR
- bfe47c8f-331e-39a4-b23e-1a7eb2373e40
-
- 0 sec
- 1
- DO_NOT_COMPRESS
-
- LOAD_BALANCE_NOT_CONFIGURED
- DO_NOT_LOAD_BALANCE
-
-
- 0
-
79b60de4-815c-3c57-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -923,16 +1092,16 @@
0
- 7aa3f1f6-cdb2-3347-0000-000000000000
+ 7d6c3686-b25c-375e-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 7aa3f1f6-cdb2-3347-8ed0-f2e41db2d520
+ 7d6c3686-b25c-375e-90c7-49397962a01b
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- a481ce78-a522-3484-0000-000000000000
+ 7b107f49-071a-3b24-0000-000000000000
FUNNEL
- a481ce78-a522-3484-9035-75535afddc3d
+ 7b107f49-071a-3b24-a4a5-b04732fd9431
0 sec
1
@@ -1063,34 +1232,6 @@
0
-
- 94d35ecb-f4f5-390e-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
- 94d35ecb-f4f5-390e-bc8a-894bfa97dba4
- 1 GB
- 10000
-
- ba40cb68-0d02-3fdf-0000-000000000000
- 7dd60db1-2d31-351c-0000-000000000000
- PROCESSOR
- 7dd60db1-2d31-351c-89c1-c2ed37eb4567
-
- 0 sec
- 1
- DO_NOT_COMPRESS
-
- LOAD_BALANCE_NOT_CONFIGURED
- DO_NOT_LOAD_BALANCE
-
- success
-
- 0
-
a5d103fd-4503-304b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -1148,16 +1289,16 @@
0
- b38a3841-5a8e-375d-0000-000000000000
+ aec23fab-963a-3b5f-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- b38a3841-5a8e-375d-927e-5e201cd15e0e
+ aec23fab-963a-3b5f-b1c7-7ddcc1dc69ef
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 2c150f1f-1273-3eab-0000-000000000000
- PROCESSOR
- 2c150f1f-1273-3eab-91f0-4e39828ec66d
+ d2f19026-1f7f-3dc1-0000-000000000000
+ FUNNEL
+ d2f19026-1f7f-3dc1-9fb6-c9db68efd081
0 sec
1
@@ -1166,12 +1307,12 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- success
+ failure
0
@@ -1259,34 +1400,6 @@
0
-
- c531e317-4a31-35c2-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
- c531e317-4a31-35c2-bbe4-0e049e36ed4e
- 1 GB
- 10000
-
- ba40cb68-0d02-3fdf-0000-000000000000
- f7e901b4-6170-3e9e-0000-000000000000
- FUNNEL
- f7e901b4-6170-3e9e-ad0e-469ef2d3b258
-
- 0 sec
- 1
- DO_NOT_COMPRESS
-
- LOAD_BALANCE_NOT_CONFIGURED
- DO_NOT_LOAD_BALANCE
-
- failure
-
- 0
-
c915c671-9dff-3ee1-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -1427,34 +1540,6 @@
0
-
- dc7a538f-8bba-31fd-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
- dc7a538f-8bba-31fd-922a-fee798634dad
- 1 GB
- 10000
-
- ba40cb68-0d02-3fdf-0000-000000000000
- 3d9a51b4-f3ec-3b80-0000-000000000000
- FUNNEL
- 3d9a51b4-f3ec-3b80-a4df-bafa1a13dcc3
-
- 0 sec
- 1
- DO_NOT_COMPRESS
-
- LOAD_BALANCE_NOT_CONFIGURED
- DO_NOT_LOAD_BALANCE
-
- failure
-
- 0
-
dfeab944-9f0f-3f9b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -1483,34 +1568,6 @@
0
-
- e546f45f-76e9-3682-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
- e546f45f-76e9-3682-9405-f82f6822905f
- 1 GB
- 10000
-
- ba40cb68-0d02-3fdf-0000-000000000000
- 8990354d-6606-3294-0000-000000000000
- PROCESSOR
- 8990354d-6606-3294-a0b5-64036dd8a8dd
-
- 0 sec
- 1
- DO_NOT_COMPRESS
-
- LOAD_BALANCE_NOT_CONFIGURED
- DO_NOT_LOAD_BALANCE
-
- merged
-
- 0
-
e640123e-240d-3ef8-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -1595,6 +1652,34 @@
0
+
+ f9476042-eb49-3f0d-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ f9476042-eb49-3f0d-a2cd-1b00c2f6a7b2
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 296f681d-d071-3e28-0000-000000000000
+ PROCESSOR
+ 296f681d-d071-3e28-9df4-326348c46644
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ 0
+
ff2066bb-9e28-3d09-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -1867,13 +1952,13 @@
2fe3e4dc-fc87-36bb-aa4f-f0a00544fc47
- 3d9a51b4-f3ec-3b80-0000-000000000000
+ 3fd696b4-944f-3364-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1240.0
- 2440.0
+ -2904.0
+ 2528.0
- 3d9a51b4-f3ec-3b80-a4df-bafa1a13dcc3
+ 3fd696b4-944f-3364-b0a8-09968605fe97
72a4a83e-464d-30e0-0000-000000000000
@@ -1885,21 +1970,22 @@
72a4a83e-464d-30e0-ae62-5ffff848e2ff
- 8770d639-3831-3469-0000-000000000000
+ 7b107f49-071a-3b24-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -2988.8388591047024
- 1679.8518483483704
+ -2424.0
+ 2192.0
+ 7b107f49-071a-3b24-a4a5-b04732fd9431
- a481ce78-a522-3484-0000-000000000000
+ 8770d639-3831-3469-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -2432.0
- 2152.0
+ -2988.8388591047024
+ 1679.8518483483704
- a481ce78-a522-3484-9035-75535afddc3d
+ 8770d639-3831-3469-be8b-b1bfb2976815
a5436bd7-26bb-30bb-0000-000000000000
@@ -1923,8 +2009,8 @@
c87baaa6-745a-3011-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1000.0
- 2328.0
+ -904.0
+ 2296.0
c87baaa6-745a-3011-ae20-ebaac0428cfe
@@ -1937,6 +2023,15 @@
d1e53967-c2ec-394e-8527-eee1e3c8a36e
+
+ d2f19026-1f7f-3dc1-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1824.0
+ 2888.0
+
+ d2f19026-1f7f-3dc1-9fb6-c9db68efd081
+
da0f9bf1-0677-36c9-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -1973,15 +2068,6 @@
f055188a-7690-3670-a19f-5f66540f17bd
-
- f7e901b4-6170-3e9e-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
-
- -2432.0
- 2704.0
-
- f7e901b4-6170-3e9e-ad0e-469ef2d3b258
-
fffda609-5de0-3e17-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -2107,8 +2193,8 @@ delimited by ";".
635d2ddc-3a65-3ee6-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1432.0
- 2240.0
+ -1344.0
+ 2048.0
635d2ddc-3a65-3ee6-8b4a-98c83afb7752
40.0
@@ -2171,6 +2257,7 @@ Also the change index name.
-4176.0
1616.0
+ bef4dbe7-7d21-3430-9e89-e1d42d1ab83b
144.0
false
ReplaceText-removeBrackets
- false
- failure
+ false
+ failure
+ false
+
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.ReplaceText
+
+
+ 1b0dc321-d6a7-3bfa-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -192.0
+ 1912.0
+
+ 1b0dc321-d6a7-3bfa-a9ec-d184f3150182
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.25.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Byte Sequence Format
+
+ Byte Sequence Format
+
+
+
+Byte Sequence
+
+ Byte Sequence
+
+
+
+Keep Byte Sequence
+
+ Keep Byte Sequence
+
+
+
+Byte Sequence Location
+
+ Byte Sequence Location
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Byte Sequence Format
+Text
+
+
+Byte Sequence
+,
+
+
+Keep Byte Sequence
+false
+
+
+Byte Sequence Location
+Trailing
+
+
+ 10
+ 0
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ SplitContent
+
+ true
+ original
false
false
- success
+ splits
false
STOPPED
- org.apache.nifi.processors.standard.ReplaceText
+ org.apache.nifi.processors.standard.SplitContent
- 1b0dc321-d6a7-3bfa-0000-000000000000
+ 296f681d-d071-3e28-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -192.0
- 1912.0
+ -2584.0
+ 2864.0
- 1b0dc321-d6a7-3bfa-a9ec-d184f3150182
+ 296f681d-d071-3e28-9df4-326348c46644
- nifi-standard-nar
+ nifi-update-attribute-nar
org.apache.nifi
1.25.0
@@ -2432,27 +2607,39 @@ and scheduling if you have less files
1
-Byte Sequence Format
+Delete Attributes Expression
- Byte Sequence Format
+ Delete Attributes Expression
-Byte Sequence
+Store State
- Byte Sequence
+ Store State
-Keep Byte Sequence
+Stateful Variables Initial Value
- Keep Byte Sequence
+ Stateful Variables Initial Value
-Byte Sequence Location
+canonical-value-lookup-cache-size
- Byte Sequence Location
+ canonical-value-lookup-cache-size
+
+
+
+filename
+
+ filename
+
+
+
+output_filename
+
+ output_filename
@@ -2462,43 +2649,44 @@ and scheduling if you have less files
30 sec
-Byte Sequence Format
-Text
+Delete Attributes Expression
-Byte Sequence
-,
+Store State
+Do not store state
-Keep Byte Sequence
-false
+Stateful Variables Initial Value
-Byte Sequence Location
-Trailing
+canonical-value-lookup-cache-size
+100
+
+
+filename
+${annotations_index_name:append("__"):append(${timestamp}):append("__"):append(${ann_timestamp}):append(".json")}
+
+
+output_filename
+${annotations_index_name:append("__"):append(${timestamp}):append("__"):append(${ann_timestamp}):append(".json")}
10
- 0
+ 25
0 sec
TIMER_DRIVEN
1 sec
false
- SplitContent
-
- true
- original
- false
-
+ UpdateAttribute-filename-annotations
false
- splits
+ success
false
STOPPED
- org.apache.nifi.processors.standard.SplitContent
+ org.apache.nifi.processors.attributes.UpdateAttribute
2c150f1f-1273-3eab-0000-000000000000
@@ -2887,6 +3075,97 @@ and scheduling if you have less files
org.apache.nifi.processors.attributes.UpdateAttribute
+
+ 440d39c2-9205-3aaf-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -2584.0
+ 2688.0
+
+ 440d39c2-9205-3aaf-8def-38cc36b5a733
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.25.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Delete Attributes Expression
+
+ Delete Attributes Expression
+
+
+
+Store State
+
+ Store State
+
+
+
+Stateful Variables Initial Value
+
+ Stateful Variables Initial Value
+
+
+
+canonical-value-lookup-cache-size
+
+ canonical-value-lookup-cache-size
+
+
+
+ann_timestamp
+
+ ann_timestamp
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Delete Attributes Expression
+
+
+Store State
+Do not store state
+
+
+Stateful Variables Initial Value
+
+
+canonical-value-lookup-cache-size
+1
+
+
+ann_timestamp
+${now():format('yyyy_MM_dd_HH_mm_ss_SSS')}
+
+
+ 10
+ 0
+ 1 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-addTimestampAnn
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
56582913-7c21-31da-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -3251,6 +3530,97 @@ and scheduling if you have less files
org.apache.nifi.processors.standard.MergeContent
+
+ 59dda431-fb7c-3a65-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1408.0
+ 2304.0
+
+ 59dda431-fb7c-3a65-b2d7-1b0980c0b915
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.25.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Delete Attributes Expression
+
+ Delete Attributes Expression
+
+
+
+Store State
+
+ Store State
+
+
+
+Stateful Variables Initial Value
+
+ Stateful Variables Initial Value
+
+
+
+canonical-value-lookup-cache-size
+
+ canonical-value-lookup-cache-size
+
+
+
+fail_flag
+
+ fail_flag
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Delete Attributes Expression
+
+
+Store State
+Do not store state
+
+
+Stateful Variables Initial Value
+
+
+canonical-value-lookup-cache-size
+1
+
+
+fail_flag
+true
+
+
+ 10
+ 0
+ 1 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-addFailFlag
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
5be72191-ed94-3f80-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
@@ -3732,8 +4102,8 @@ and scheduling if you have less files
7dd60db1-2d31-351c-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1984.0
- 2488.0
+ -1976.0
+ 2632.0
7dd60db1-2d31-351c-89c1-c2ed37eb4567
@@ -4619,7 +4989,7 @@ and scheduling if you have less files
10
0
- 10 sec
+ 5 sec
TIMER_DRIVEN
1 sec
@@ -4756,7 +5126,7 @@ and scheduling if you have less files
bfe47c8f-331e-39a4-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1392.0
+ -1400.0
2096.0
bfe47c8f-331e-39a4-b23e-1a7eb2373e40
@@ -5259,8 +5629,8 @@ and scheduling if you have less files
e7943295-e173-3870-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1984.0
- 2088.0
+ -1992.0
+ 2096.0
e7943295-e173-3870-9c6c-c7357e0fb449
@@ -5531,7 +5901,7 @@ and scheduling if you have less files
ef208086-8b96-38e5-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1984.0
+ -1992.0
2296.0
ef208086-8b96-38e5-86a2-d12f3df800ec
@@ -5837,7 +6207,7 @@ and scheduling if you have less files
success
false
- STOPPED
+ DISABLED
org.apache.nifi.processors.standard.ExecuteProcess
@@ -5852,5 +6222,5 @@ and scheduling if you have less files
- 05/24/2024 11:56:56 BST
+ 05/31/2024 09:34:51 BST