I got there in the end, chat gpt did eventually work for me, after a bit of debugging and error handling.
For anyone in the same boat, hopefully this code can help.
let
// Define the Table.GenerateByPage function for pagination
Table.GenerateByPage = (getNextPage as function) as table =>
let
listOfPages = List.Generate(
() => getNextPage(0), // Start with the first page (skip=0)
(lastPage) => lastPage <> null, // Stop when null is returned (no more data)
(lastPage) => getNextPage(Table.RowCount(lastPage)) // Pass the row count as the next skip value
),
// Concatenate the pages together
tableOfPages = Table.FromList(listOfPages, Splitter.SplitByNothing(), {"Column1"}),
firstRow = tableOfPages{0}?
in
// If no pages are returned, return an empty table
if (firstRow = null) then
Table.FromRows({})
else if (Table.IsEmpty(firstRow[Column1])) then
firstRow[Column1]
else
Value.ReplaceType(
Table.ExpandTableColumn(tableOfPages, "Column1", Table.ColumnNames(firstRow[Column1])),
Value.Type(firstRow[Column1])
),
// Define the getNextPage function with enhanced error handling
getNextPage = (skipCount as number) =>
let
// Construct the API URL with pagination logic
url = "https://finder.bloodsandbeyond.co.uk/myURL?$skip=" & Number.ToText(skipCount),
headers = [
#"x-api-token" = "?????????????????????????"
],
source = try Web.Contents(url, [Headers=headers]) otherwise null,
// Convert the response to text and parse it as JSON
rawResponse = if source = null then error "No data from API!" else Text.FromBinary(source),
jsonResponse = try Json.Document(rawResponse) otherwise null,
// Check if the "value" field exists and is valid
value = if jsonResponse = null or not Record.HasFields(jsonResponse, "value") then null else jsonResponse[value],
// If value is null or empty, stop fetching further pages
nextPage = if value = null or List.IsEmpty(value) then null else value
in
// Return the result as a table or null
if nextPage = null then null else Table.FromList(nextPage, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
// Use Table.GenerateByPage to get all data
allData = Table.GenerateByPage(getNextPage)
in
allData