From 9d78f26814088a53a125ed55ead32f8ebc2f3db3 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 14:56:50 +0100 Subject: [PATCH 1/9] Increase default pagination to 100 --- app/api/api/paginations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/api/paginations.py b/app/api/api/paginations.py index 987a14087..cf96ce46c 100644 --- a/app/api/api/paginations.py +++ b/app/api/api/paginations.py @@ -4,5 +4,5 @@ class CustomPagination(pagination.PageNumberPagination): page_size = 10 page_size_query_param = "page_size" - max_page_size = 50 + max_page_size = 100 page_query_param = "p" From 8f4a9bb8062ab53127b0e70c6d58514543e9be52 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 14:57:55 +0100 Subject: [PATCH 2/9] Set default on table update --- .../app/scanreports/[id]/tables/[tableId]/update/page.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx b/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx index 089839999..23d133283 100644 --- a/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx +++ b/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx @@ -26,9 +26,11 @@ interface UpdateTableProps { export default async function UpdateTable({ params: { id, tableId }, }: UpdateTableProps) { + const defaultPageSize = 100; const defaultParams = { scan_report_table: tableId, fields: "name,id", + page_size: defaultPageSize, }; const combinedParams = { ...defaultParams }; @@ -39,7 +41,7 @@ export default async function UpdateTable({ (item: ScanReportField) => ({ id: item.id, name: item.name, - }) + }), ); const scanReportsName = await getScanReport(id); const table = await getScanReportTable(tableId); From 391202ce8d3e4eabd92e1f0377c7222308f86efb Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 15:46:27 +0100 Subject: [PATCH 3/9] Revert max_page_size --- app/api/api/paginations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/api/paginations.py b/app/api/api/paginations.py index cf96ce46c..987a14087 100644 --- a/app/api/api/paginations.py +++ b/app/api/api/paginations.py @@ -4,5 +4,5 @@ class CustomPagination(pagination.PageNumberPagination): page_size = 10 page_size_query_param = "page_size" - max_page_size = 100 + max_page_size = 50 page_query_param = "p" From 97753389b2466f7fddda51d3e537c611456a0ee5 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 15:46:41 +0100 Subject: [PATCH 4/9] Add utils function --- app/next-client-app/lib/api/utils.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 app/next-client-app/lib/api/utils.ts diff --git a/app/next-client-app/lib/api/utils.ts b/app/next-client-app/lib/api/utils.ts new file mode 100644 index 000000000..5ecabeebe --- /dev/null +++ b/app/next-client-app/lib/api/utils.ts @@ -0,0 +1,24 @@ +import request from "@/lib/api/request"; + +/** + * Fetches all pages of paginated data from the specified initial URL. + * + * Iterates through paginated data until all pages are fetched and concatenated. + * + * @template T - The type of data to be fetched + * @param {string} initialUrl - The initial URL to start fetching paginated data from + * @returns {Promise} - A promise that resolves to an array of all fetched data + */ +export async function fetchAllPages(initialUrl: string): Promise { + let url: string | null = initialUrl; + let allResults: T[] = []; + + while (url) { + const data: PaginatedResponse = await request(url); + + allResults = allResults.concat(data.results); + url = data.next; + } + + return allResults; +} From 47d4f041eb54eb5836a5e3bed276886199000226 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 15:46:55 +0100 Subject: [PATCH 5/9] Implement `getAllScanReportFields` --- app/next-client-app/api/scanreports.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/next-client-app/api/scanreports.ts b/app/next-client-app/api/scanreports.ts index d370492af..171ede17a 100644 --- a/app/next-client-app/api/scanreports.ts +++ b/app/next-client-app/api/scanreports.ts @@ -2,6 +2,7 @@ import { revalidatePath } from "next/cache"; import request from "@/lib/api/request"; import { redirect } from "next/navigation"; +import { fetchAllPages } from "@/lib/api/utils"; const fetchKeys = { list: (filter?: string) => @@ -72,6 +73,17 @@ export async function getScanReportFields( } } +export async function getAllScanReportFields( + filter: string | undefined, +): Promise { + try { + return await fetchAllPages(fetchKeys.fields(filter)); + } catch (error) { + console.warn("Failed to fetch data."); + return []; + } +} + export async function getScanReportValues( filter: string | undefined, ): Promise> { From a88a554e467a263ce0589766432aeecec7702356 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 15:47:16 +0100 Subject: [PATCH 6/9] Fix on page level --- .../[id]/tables/[tableId]/update/page.tsx | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx b/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx index 23d133283..adb43b51d 100644 --- a/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx +++ b/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx @@ -6,8 +6,8 @@ import { BreadcrumbSeparator, } from "@/components/ui/breadcrumb"; import { + getAllScanReportFields, getScanReport, - getScanReportFields, getScanReportPermissions, getScanReportTable, } from "@/api/scanreports"; @@ -36,13 +36,11 @@ export default async function UpdateTable({ const query = objToQuery(combinedParams); - const scanReportsFields = await getScanReportFields(query); - const shortenFields = scanReportsFields.results.map( - (item: ScanReportField) => ({ - id: item.id, - name: item.name, - }), - ); + const scanReportsFields = await getAllScanReportFields(query); + const shortenFields = scanReportsFields.map((item: ScanReportField) => ({ + id: item.id, + name: item.name, + })); const scanReportsName = await getScanReport(id); const table = await getScanReportTable(tableId); const permissions = await getScanReportPermissions(id); From 96d0daeb11cfc20cd424c7bbc5f727bb98e81332 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 15:52:26 +0100 Subject: [PATCH 7/9] Implement default size --- .../app/scanreports/[id]/tables/[tableId]/update/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx b/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx index adb43b51d..92ad277a4 100644 --- a/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx +++ b/app/next-client-app/app/scanreports/[id]/tables/[tableId]/update/page.tsx @@ -26,7 +26,7 @@ interface UpdateTableProps { export default async function UpdateTable({ params: { id, tableId }, }: UpdateTableProps) { - const defaultPageSize = 100; + const defaultPageSize = 50; const defaultParams = { scan_report_table: tableId, fields: "name,id", From b1816b2c33b9861bc598f69bab2a7d092184bcfa Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 16:16:35 +0100 Subject: [PATCH 8/9] Fix utils --- app/next-client-app/lib/api/utils.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/next-client-app/lib/api/utils.ts b/app/next-client-app/lib/api/utils.ts index 5ecabeebe..817c287cc 100644 --- a/app/next-client-app/lib/api/utils.ts +++ b/app/next-client-app/lib/api/utils.ts @@ -1,23 +1,25 @@ import request from "@/lib/api/request"; /** - * Fetches all pages of paginated data from the specified initial URL. + * Fetches all pages of paginated data from the specified url. * * Iterates through paginated data until all pages are fetched and concatenated. * * @template T - The type of data to be fetched - * @param {string} initialUrl - The initial URL to start fetching paginated data from + * @param {string} url - The url to fetch from * @returns {Promise} - A promise that resolves to an array of all fetched data */ -export async function fetchAllPages(initialUrl: string): Promise { - let url: string | null = initialUrl; +export async function fetchAllPages(url: string): Promise { let allResults: T[] = []; + let pageNumber = 1; - while (url) { - const data: PaginatedResponse = await request(url); + while (true) { + const urlWithPage = `${url}&p=${pageNumber}`; + const data: PaginatedResponse = await request(urlWithPage); allResults = allResults.concat(data.results); - url = data.next; + pageNumber++; + if (!data.next) break; } return allResults; From e9eaf08d457adbb7a0927ec77a7dfbed0227a2d3 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Sat, 22 Jun 2024 16:23:47 +0100 Subject: [PATCH 9/9] Fix import --- app/next-client-app/lib/api/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/next-client-app/lib/api/utils.ts b/app/next-client-app/lib/api/utils.ts index 817c287cc..2a6dba16d 100644 --- a/app/next-client-app/lib/api/utils.ts +++ b/app/next-client-app/lib/api/utils.ts @@ -1,4 +1,4 @@ -import request from "@/lib/api/request"; +import request from "./request"; /** * Fetches all pages of paginated data from the specified url.