JavaScript Pagination¶
Advanced pagination strategies for efficiently handling large datasets from the Danish Parliament API using modern JavaScript patterns.
Overview¶
The API limits requests to 100 records maximum. For large datasets (96,538+ cases, 18,139+ actors), you need efficient pagination strategies using JavaScript generators, async iterators, and concurrent processing.
Basic Pagination Patterns¶
1. Simple Loop-Based Pagination¶
import { DanishParliamentAPI } from './danish-parliament-api.js';
/**
* Fetch all records using traditional pagination
*/
async function fetchAllRecords(api, entity, maxRecords = Infinity) {
const allRecords = [];
let skip = 0;
const batchSize = 100;
while (allRecords.length < maxRecords && skip < 100000) {
console.log(`Fetching batch at skip=${skip}...`);
const response = await api.request(entity, {
'$top': batchSize,
'$skip': skip
});
const records = response.value || [];
if (records.length === 0) {
break; // No more data
}
allRecords.push(...records);
skip += batchSize;
// Progress logging
if (allRecords.length % 1000 === 0) {
console.log(`Fetched ${allRecords.length.toLocaleString()} records...`);
}
// Rate limiting - be respectful to the API
await new Promise(resolve => setTimeout(resolve, 100));
}
return allRecords.slice(0, maxRecords);
}
// Usage
const api = new DanishParliamentAPI();
console.log('Fetching all climate cases...');
const climateCases = await fetchAllRecords(api, 'Sag', 1000); // Limit for demo
console.log(`Total climate cases: ${climateCases.length}`);
2. Generator-Based Pagination (Recommended)¶
Memory-efficient approach using async generators:
/**
* Async generator for memory-efficient pagination
*
* @param {DanishParliamentAPI} api - API client instance
* @param {string} entity - Entity name
* @param {Object} options - Pagination options
*/
async function* paginateRecords(api, entity, options = {}) {
const {
batchSize = 100,
maxRecords = Infinity,
filter = null,
expand = null,
select = null
} = options;
let skip = 0;
let totalYielded = 0;
while (totalYielded < maxRecords && skip < 100000) {
const params = {
'$top': Math.min(batchSize, 100),
'$skip': skip
};
if (filter) params['$filter'] = filter;
if (expand) params['$expand'] = expand;
if (select) params['$select'] = select;
try {
const response = await api.request(entity, params);
const records = response.value || [];
if (records.length === 0) {
break; // No more data
}
// Yield each record individually
for (const record of records) {
if (totalYielded >= maxRecords) {
return; // Reached limit
}
yield record;
totalYielded++;
}
skip += batchSize;
// Rate limiting
await new Promise(resolve => setTimeout(resolve, 100));
} catch (error) {
console.error(`Error at skip=${skip}:`, error.message);
break;
}
}
}
// Usage with for-await-of
const api = new DanishParliamentAPI();
console.log('Processing all climate cases...');
let processedCount = 0;
for await (const case of paginateRecords(api, 'Sag', {
filter: "substringof('klima', titel)",
maxRecords: 500 // Demo limit
})) {
processedCount++;
// Process each case individually without storing all in memory
console.log(`${processedCount}: ${case.titel.substring(0, 50)}...`);
// Your processing logic here:
// - Save to database
// - Transform data
// - Analyze content
// - etc.
}
console.log(`Finished processing ${processedCount} climate cases`);
3. Concurrent Batch Processing¶
For faster data retrieval using parallel requests:
/**
* Fetch multiple batches concurrently for faster processing
*/
class ConcurrentPaginator {
constructor(api, maxConcurrent = 5) {
this.api = api;
this.maxConcurrent = maxConcurrent;
}
/**
* Create a semaphore to limit concurrent requests
*/
createSemaphore(max) {
const semaphore = {
current: 0,
queue: [],
async acquire() {
return new Promise(resolve => {
if (this.current < max) {
this.current++;
resolve();
} else {
this.queue.push(resolve);
}
});
},
release() {
this.current--;
if (this.queue.length > 0) {
this.current++;
const resolve = this.queue.shift();
resolve();
}
}
};
return semaphore;
}
/**
* Fetch data using concurrent batches
*/
async fetchConcurrentBatches(entity, totalRecords, options = {}) {
const { batchSize = 100, filter, expand, select } = options;
const semaphore = this.createSemaphore(this.maxConcurrent);
// Calculate batch positions
const batches = [];
for (let skip = 0; skip < totalRecords; skip += batchSize) {
batches.push({
skip,
size: Math.min(batchSize, totalRecords - skip)
});
}
console.log(`Fetching ${batches.length} batches concurrently (max ${this.maxConcurrent} at once)...`);
// Fetch function for a single batch
const fetchBatch = async (batch) => {
await semaphore.acquire();
try {
const params = {
'$top': batch.size,
'$skip': batch.skip
};
if (filter) params['$filter'] = filter;
if (expand) params['$expand'] = expand;
if (select) params['$select'] = select;
const response = await this.api.request(entity, params);
console.log(`✅ Batch at skip=${batch.skip}: ${response.value?.length || 0} records`);
return {
skip: batch.skip,
records: response.value || []
};
} catch (error) {
console.error(`L Batch failed at skip=${batch.skip}:`, error.message);
return {
skip: batch.skip,
records: [],
error: error.message
};
} finally {
semaphore.release();
}
};
// Execute all batches concurrently
const startTime = Date.now();
const results = await Promise.all(batches.map(fetchBatch));
const endTime = Date.now();
// Sort results by skip position and flatten
results.sort((a, b) => a.skip - b.skip);
const allRecords = results.flatMap(result => result.records);
console.log(`\nConcurrent fetch completed:`);
console.log(`- Time: ${((endTime - startTime) / 1000).toFixed(2)} seconds`);
console.log(`- Records: ${allRecords.length.toLocaleString()}`);
console.log(`- Speed: ${Math.round(allRecords.length / ((endTime - startTime) / 1000))} records/second`);
return allRecords;
}
}
// Usage
const api = new DanishParliamentAPI();
const paginator = new ConcurrentPaginator(api, 3); // Max 3 concurrent requests
// Fetch climate cases using concurrent batches
const climateCases = await paginator.fetchConcurrentBatches('Sag', 500, {
filter: "substringof('klima', titel)"
});
console.log(`Retrieved ${climateCases.length} climate cases`);
Advanced Pagination Patterns¶
1. Resumable Pagination with Progress Tracking¶
/**
* Pagination with progress tracking and resumption capability
*/
class ResumablePaginator {
constructor(api, entity, options = {}) {
this.api = api;
this.entity = entity;
this.options = options;
this.progressKey = `pagination_${entity}_${Date.now()}`;
this.progress = this.loadProgress();
}
// Load progress from localStorage (browser) or file (Node.js)
loadProgress() {
try {
if (typeof localStorage !== 'undefined') {
// Browser environment
const saved = localStorage.getItem(this.progressKey);
return saved ? JSON.parse(saved) : { skip: 0, totalFetched: 0 };
} else {
// Node.js environment - would need fs module
return { skip: 0, totalFetched: 0 };
}
} catch (error) {
console.warn('Could not load progress:', error.message);
return { skip: 0, totalFetched: 0 };
}
}
// Save progress
saveProgress() {
try {
if (typeof localStorage !== 'undefined') {
localStorage.setItem(this.progressKey, JSON.stringify(this.progress));
}
// In Node.js, you could save to a file
} catch (error) {
console.warn('Could not save progress:', error.message);
}
}
// Paginate with resume capability
async* paginateWithResume(maxRecords = Infinity) {
let skip = this.progress.skip;
let totalYielded = this.progress.totalFetched;
const batchSize = 100;
console.log(`Resuming from skip=${skip}, totalFetched=${totalYielded}`);
while (totalYielded < maxRecords && skip < 100000) {
try {
const params = {
'$top': batchSize,
'$skip': skip,
...this.options
};
const response = await this.api.request(this.entity, params);
const records = response.value || [];
if (records.length === 0) {
break;
}
// Update and save progress
this.progress.skip = skip + batchSize;
this.progress.totalFetched = totalYielded + records.length;
this.saveProgress();
// Yield records
for (const record of records) {
if (totalYielded >= maxRecords) return;
yield record;
totalYielded++;
}
skip += batchSize;
// Progress reporting
if (totalYielded % 500 === 0) {
console.log(`Progress: ${totalYielded.toLocaleString()} records processed`);
}
// Rate limiting
await new Promise(resolve => setTimeout(resolve, 100));
} catch (error) {
console.error(`Error at skip=${skip}:`, error.message);
console.log('Progress saved. You can resume later by creating a new paginator.');
throw error;
}
}
// Clear progress on completion
this.clearProgress();
}
clearProgress() {
try {
if (typeof localStorage !== 'undefined') {
localStorage.removeItem(this.progressKey);
}
} catch (error) {
console.warn('Could not clear progress:', error.message);
}
}
}
// Usage
const api = new DanishParliamentAPI();
const paginator = new ResumablePaginator(api, 'Sag', {
'$filter': "substringof('klima', titel)"
});
try {
let count = 0;
for await (const case of paginator.paginateWithResume(1000)) {
count++;
console.log(`${count}: Processing case ${case.id}`);
// Your processing logic here
// Simulate interruption for demo
if (count === 250) {
throw new Error('Simulated interruption');
}
}
} catch (error) {
console.log('Interrupted:', error.message);
console.log('Run the code again to resume from where you left off');
}
2. Smart Pagination with Dynamic Batching¶
/**
* Smart paginator that adjusts batch size based on response times
*/
class SmartPaginator {
constructor(api, targetResponseTime = 1000) {
this.api = api;
this.targetResponseTime = targetResponseTime; // Target 1 second per request
this.currentBatchSize = 100;
this.responseTimeHistory = [];
}
// Adjust batch size based on response times
adjustBatchSize(responseTime) {
this.responseTimeHistory.push(responseTime);
// Keep only last 10 response times
if (this.responseTimeHistory.length > 10) {
this.responseTimeHistory.shift();
}
// Calculate average response time
const avgResponseTime = this.responseTimeHistory.reduce((a, b) => a + b, 0) / this.responseTimeHistory.length;
if (avgResponseTime > this.targetResponseTime && this.currentBatchSize > 10) {
// Too slow - decrease batch size
this.currentBatchSize = Math.max(10, Math.floor(this.currentBatchSize * 0.8));
console.log(`Decreased batch size to ${this.currentBatchSize} (avg response time: ${Math.round(avgResponseTime)}ms)`);
} else if (avgResponseTime < this.targetResponseTime * 0.5 && this.currentBatchSize < 100) {
// Very fast - increase batch size
this.currentBatchSize = Math.min(100, Math.floor(this.currentBatchSize * 1.2));
console.log(`Increased batch size to ${this.currentBatchSize} (avg response time: ${Math.round(avgResponseTime)}ms)`);
}
}
async* smartPaginate(entity, options = {}) {
let skip = 0;
let totalYielded = 0;
while (skip < 100000) { // Safety limit
const startTime = Date.now();
try {
const params = {
'$top': this.currentBatchSize,
'$skip': skip,
...options
};
const response = await this.api.request(entity, params);
const records = response.value || [];
const responseTime = Date.now() - startTime;
this.adjustBatchSize(responseTime);
if (records.length === 0) {
break;
}
// Yield records
for (const record of records) {
yield record;
totalYielded++;
}
skip += this.currentBatchSize;
// Progress reporting
if (totalYielded % 200 === 0) {
console.log(`Smart pagination: ${totalYielded} records, batch size: ${this.currentBatchSize}`);
}
// Adaptive delay based on response time
const delay = Math.min(200, Math.max(50, responseTime * 0.1));
await new Promise(resolve => setTimeout(resolve, delay));
} catch (error) {
console.error(`Smart pagination error at skip=${skip}:`, error.message);
// Reduce batch size on error
this.currentBatchSize = Math.max(10, Math.floor(this.currentBatchSize * 0.5));
console.log(`Reduced batch size to ${this.currentBatchSize} due to error`);
skip += this.currentBatchSize; // Skip problematic batch
}
}
}
}
// Usage
const api = new DanishParliamentAPI();
const smartPaginator = new SmartPaginator(api, 800); // Target 800ms response time
let count = 0;
for await (const case of smartPaginator.smartPaginate('Sag', {
'$filter': "year(opdateringsdato) eq 2025"
})) {
count++;
console.log(`${count}: ${case.titel.substring(0, 40)}...`);
if (count >= 300) break; // Demo limit
}
Real-World Usage Examples¶
1. ETL Pipeline with Error Recovery¶
/**
* Complete ETL pipeline with error recovery and monitoring
*/
class ETLPipeline {
constructor(api, options = {}) {
this.api = api;
this.options = {
batchSize: 100,
maxRetries: 3,
retryDelay: 1000,
...options
};
this.stats = {
processed: 0,
errors: 0,
startTime: null,
endTime: null
};
}
async processEntity(entity, processor, filter = null) {
this.stats.startTime = Date.now();
console.log(`Starting ETL pipeline for ${entity}...`);
try {
for await (const record of this.paginateWithRetry(entity, filter)) {
try {
// Process individual record
await processor(record);
this.stats.processed++;
// Progress reporting
if (this.stats.processed % 100 === 0) {
const elapsed = (Date.now() - this.stats.startTime) / 1000;
const rate = this.stats.processed / elapsed;
console.log(`Processed: ${this.stats.processed}, Rate: ${rate.toFixed(1)} records/sec`);
}
} catch (error) {
this.stats.errors++;
console.error(`Error processing record ${record.id}:`, error.message);
// Continue processing despite individual record errors
}
}
} finally {
this.stats.endTime = Date.now();
this.logFinalStats();
}
}
async* paginateWithRetry(entity, filter) {
let skip = 0;
while (skip < 100000) {
let retryCount = 0;
while (retryCount < this.options.maxRetries) {
try {
const params = {
'$top': this.options.batchSize,
'$skip': skip
};
if (filter) params['$filter'] = filter;
const response = await this.api.request(entity, params);
const records = response.value || [];
if (records.length === 0) {
return; // No more data
}
// Yield all records from this batch
for (const record of records) {
yield record;
}
break; // Success - exit retry loop
} catch (error) {
retryCount++;
console.warn(`Batch failed at skip=${skip}, retry ${retryCount}/${this.options.maxRetries}: ${error.message}`);
if (retryCount >= this.options.maxRetries) {
console.error(`Giving up on batch at skip=${skip} after ${this.options.maxRetries} retries`);
break; // Move to next batch
}
// Exponential backoff
const delay = this.options.retryDelay * Math.pow(2, retryCount - 1);
await new Promise(resolve => setTimeout(resolve, delay));
}
}
skip += this.options.batchSize;
}
}
logFinalStats() {
const elapsed = (this.stats.endTime - this.stats.startTime) / 1000;
const rate = this.stats.processed / elapsed;
console.log('\n=== ETL Pipeline Complete ==🔧);
console.log(`Total processed: ${this.stats.processed.toLocaleString()}`);
console.log(`Total errors: ${this.stats.errors}`);
console.log(`Time elapsed: ${elapsed.toFixed(1)} seconds`);
console.log(`Processing rate: ${rate.toFixed(1)} records/second`);
console.log(`Success rate: ${((this.stats.processed / (this.stats.processed + this.stats.errors)) * 100).toFixed(1)}%`);
}
}
// Usage example
const api = new DanishParliamentAPI();
const etl = new ETLPipeline(api);
// Define a record processor
const caseProcessor = async (case) => {
// Example: Extract and save key information
const processedCase = {
id: case.id,
title: case.titel,
type: case.typeid,
updated: case.opdateringsdato,
// Add your transformation logic here
};
// Save to database, file, or other destination
console.log(`Processing case: ${processedCase.title?.substring(0, 30)}...`);
// Simulate some processing time
await new Promise(resolve => setTimeout(resolve, 10));
};
// Run the ETL pipeline
await etl.processEntity('Sag', caseProcessor, "substringof('klima', titel)");
Performance Best Practices¶
1. Memory Management¶
// Good: Process records one by one
for await (const record of paginateRecords(api, 'Sag')) {
processRecord(record); // Memory usage stays constant
}
// Bad: Load everything into memory
const allRecords = await fetchAllRecords(api, 'Sag'); // Uses 100+ MB
allRecords.forEach(processRecord);
2. Rate Limiting¶
// Always include delays between requests
const RESPECTFUL_DELAY = 100; // 100ms between requests
async function respectfulPagination(api, entity) {
for (let skip = 0; skip < 10000; skip += 100) {
const response = await api.request(entity, { '$top': 100, '$skip': skip });
// Process response...
// Be respectful to the API
await new Promise(resolve => setTimeout(resolve, RESPECTFUL_DELAY));
}
}
3. Error Resilience¶
// Handle network errors gracefully
async function resilientPagination(api, entity) {
let skip = 0;
let consecutiveErrors = 0;
const MAX_CONSECUTIVE_ERRORS = 3;
while (consecutiveErrors < MAX_CONSECUTIVE_ERRORS && skip < 100000) {
try {
const response = await api.request(entity, { '$top': 100, '$skip': skip });
// Process response...
consecutiveErrors = 0; // Reset on success
skip += 100;
} catch (error) {
consecutiveErrors++;
console.warn(`Error ${consecutiveErrors}/${MAX_CONSECUTIVE_ERRORS}:`, error.message);
// Exponential backoff
const delay = Math.pow(2, consecutiveErrors) * 1000;
await new Promise(resolve => setTimeout(resolve, delay));
}
}
}
Monitoring and Analytics¶
/**
* Enhanced paginator with detailed analytics
*/
class AnalyticsPaginator {
constructor(api) {
this.api = api;
this.analytics = {
totalRequests: 0,
totalRecords: 0,
totalTime: 0,
avgResponseTime: 0,
errors: 0,
responseTimes: []
};
}
async* paginateWithAnalytics(entity, options = {}) {
const startTime = Date.now();
let skip = 0;
while (skip < 100000) {
const requestStart = Date.now();
try {
const response = await this.api.request(entity, {
'$top': 100,
'$skip': skip,
...options
});
const requestTime = Date.now() - requestStart;
this.updateAnalytics(requestTime, response.value?.length || 0);
const records = response.value || [];
if (records.length === 0) break;
for (const record of records) {
yield record;
}
skip += 100;
} catch (error) {
this.analytics.errors++;
console.error(`Request failed at skip=${skip}:`, error.message);
skip += 100; // Skip this batch
}
}
this.analytics.totalTime = Date.now() - startTime;
this.logAnalytics();
}
updateAnalytics(responseTime, recordCount) {
this.analytics.totalRequests++;
this.analytics.totalRecords += recordCount;
this.analytics.responseTimes.push(responseTime);
// Keep only last 100 response times for average calculation
if (this.analytics.responseTimes.length > 100) {
this.analytics.responseTimes.shift();
}
this.analytics.avgResponseTime =
this.analytics.responseTimes.reduce((a, b) => a + b, 0) /
this.analytics.responseTimes.length;
}
logAnalytics() {
console.log('\n=== Pagination Analytics ==🔧);
console.log(`Total requests: ${this.analytics.totalRequests}`);
console.log(`Total records: ${this.analytics.totalRecords.toLocaleString()}`);
console.log(`Total time: ${(this.analytics.totalTime / 1000).toFixed(1)}s`);
console.log(`Average response time: ${Math.round(this.analytics.avgResponseTime)}ms`);
console.log(`Records per second: ${Math.round(this.analytics.totalRecords / (this.analytics.totalTime / 1000))}`);
console.log(`Error rate: ${(this.analytics.errors / this.analytics.totalRequests * 100).toFixed(1)}%`);
}
}
These pagination patterns provide efficient, memory-safe ways to process the entire Danish Parliament dataset while being respectful to the API and handling all edge cases gracefully.