nino_mueller
12/29/2021, 12:25 PMdef get_url_params(
self, context: Optional[dict], next_page_token: Optional[Any]
) -> Dict[str, Any]:
"""Return a dictionary of values to be used in URL parameterization."""
params: dict = {}
if next_page_token:
params["offset"] = next_page_token
params["limit"] = 500
if self.replication_key:
params["sort"] = "asc"
params["order_by"] = self.replication_key
return params
Not sure what to do with `next_page_token_jsonpath = "$.next_page" # Or override `get_next_page_token``
or
def get_next_page_token(
self, response: requests.Response, previous_token: Optional[Any]
) -> Optional[Any]:
"""Return a token for identifying next page or None if no more pages."""
# TODO: If pagination is required, return a token which can be used to get the
# next page. If this is the final page, return "None" to end the
# pagination loop.
if self.next_page_token_jsonpath:
all_matches = extract_jsonpath(
self.next_page_token_jsonpath, response.json()
)
first_match = next(iter(all_matches), None)
next_page_token = first_match
else:
next_page_token = response.headers.get("X-Next-Page", None)
return next_page_token