From 98c8139c9a4e10789ed00a73ac654959569cdd05 Mon Sep 17 00:00:00 2001 From: secondl1ght Date: Sat, 25 Nov 2023 17:58:06 -0700 Subject: [PATCH] refactor sync to use incremental limit --- src/lib/sync/areas.js | 226 +++++++++++++++++++++------------- src/lib/sync/elements.js | 258 ++++++++++++++++++++++++--------------- src/lib/sync/events.js | 234 +++++++++++++++++++++-------------- src/lib/sync/reports.js | 234 +++++++++++++++++++++-------------- src/lib/sync/users.js | 226 +++++++++++++++++++++------------- 5 files changed, 723 insertions(+), 455 deletions(-) diff --git a/src/lib/sync/areas.js b/src/lib/sync/areas.js index 32458b15..197942b0 100644 --- a/src/lib/sync/areas.js +++ b/src/lib/sync/areas.js @@ -5,6 +5,8 @@ import { areas, areaError } from '$lib/store'; axiosRetry(axios, { retries: 3 }); +const limit = 500; + export const areasSync = async () => { // get areas from local await localforage @@ -12,99 +14,128 @@ export const areasSync = async () => { .then(async function (value) { // get areas from API if initial sync if (!value) { - try { - const response = await axios.get('https://api.btcmap.org/v2/areas'); - - if (response.data.length) { - // filter out deleted areas - const areasFiltered = response.data.filter((area) => !area['deleted_at']); - - // set response to local - localforage - .setItem('areas', response.data) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set response to store - areas.set(areasFiltered); - }) - .catch(function (err) { - areas.set(areasFiltered); - areaError.set( - 'Could not store areas locally, please try again or contact BTC Map.' - ); - console.log(err); - }); - } else { - areaError.set( - 'Areas API returned an empty result, please try again or contact BTC Map.' + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let areasData = []; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/areas?updated_since=${updatedSince}&limit=${limit}` ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + areasData.filter((area) => !response.data.find((data) => data.id === area.id)); + response.data.forEach((data) => areasData.push(data)); + } else { + areaError.set( + 'Areas API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + areaError.set('Could not load areas from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - areaError.set('Could not load areas from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (areasData.length) { + // filter out deleted areas + const areasFiltered = areasData.filter((area) => !area['deleted_at']); + + // set response to local + localforage + .setItem('areas', areasData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set response to store + areas.set(areasFiltered); + }) + .catch(function (err) { + areas.set(areasFiltered); + areaError.set('Could not store areas locally, please try again or contact BTC Map.'); + console.log(err); + }); } } else { // filter out deleted areas const areasFiltered = value.filter((area) => !area['deleted_at']); // start update sync from API - try { - // sort to get most recent record - let cacheSorted = [...value]; - cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + // sort to get most recent record + let cacheSorted = [...value]; + cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + + let updatedSince = cacheSorted[0]['updated_at']; + let responseCount; + let areasData = value; + let useCachedData = false; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/areas?updated_since=${updatedSince}&limit=${limit}` + ); - const response = await axios.get( - `https://api.btcmap.org/v2/areas?updated_since=${cacheSorted[0]['updated_at']}` - ); + // update new records if they exist + let newAreas = response.data; - // update new records if they exist - let newAreas = response.data; - - // check for new areas in local and purge if they exist - if (newAreas.length) { - let updatedAreas = value.filter((value) => { - if (newAreas.find((area) => area.id === value.id)) { - return false; - } else { - return true; - } - }); + // check for new areas in local and purge if they exist + if (newAreas.length) { + updatedSince = newAreas[newAreas.length - 1]['updated_at']; + responseCount = newAreas.length; - // add new areas - updatedAreas.forEach((area) => { - newAreas.push(area); - }); + areasData.filter((value) => { + if (newAreas.find((area) => area.id === value.id)) { + return false; + } else { + return true; + } + }); - // filter out deleted areas - const newAreasFiltered = newAreas.filter((area) => !area['deleted_at']); - - // set updated areas locally - localforage - .setItem('areas', newAreas) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set updated areas to store - areas.set(newAreasFiltered); - }) - .catch(function (err) { - // set updated areas to store - areas.set(newAreasFiltered); - - areaError.set( - 'Could not update areas locally, please try again or contact BTC Map.' - ); - console.log(err); + // add new areas + newAreas.forEach((area) => { + areasData.push(area); }); - } else { + } else { + // load areas from cache + areas.set(areasFiltered); + useCachedData = true; + break; + } + } catch (error) { // load areas from cache areas.set(areasFiltered); + useCachedData = true; + + areaError.set('Could not update areas from API, please try again or contact BTC Map.'); + console.error(error); + break; } - } catch (error) { - // load areas from cache - areas.set(areasFiltered); + } while (responseCount === limit); - areaError.set('Could not update areas from API, please try again or contact BTC Map.'); - console.error(error); + if (!useCachedData) { + // filter out deleted areas + const newAreasFiltered = areasData.filter((area) => !area['deleted_at']); + + // set updated areas locally + localforage + .setItem('areas', areasData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set updated areas to store + areas.set(newAreasFiltered); + }) + .catch(function (err) { + // set updated areas to store + areas.set(newAreasFiltered); + + areaError.set('Could not update areas locally, please try again or contact BTC Map.'); + console.log(err); + }); } } }) @@ -113,21 +144,40 @@ export const areasSync = async () => { areaError.set('Could not load areas locally, please try again or contact BTC Map.'); console.log(err); - try { - const response = await axios.get('https://api.btcmap.org/v2/areas'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let areasData = []; - if (response.data.length) { - // filter out deleted areas - const areasFiltered = response.data.filter((area) => !area['deleted_at']); + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/areas?updated_since=${updatedSince}&limit=${limit}` + ); - // set response to store - areas.set(areasFiltered); - } else { - areaError.set('Areas API returned an empty result, please try again or contact BTC Map.'); + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + areasData.filter((area) => !response.data.find((data) => data.id === area.id)); + response.data.forEach((data) => areasData.push(data)); + } else { + areaError.set( + 'Areas API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + areaError.set('Could not load areas from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - areaError.set('Could not load areas from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (areasData.length) { + // filter out deleted areas + const areasFiltered = areasData.filter((area) => !area['deleted_at']); + + // set response to store + areas.set(areasFiltered); } }); }; diff --git a/src/lib/sync/elements.js b/src/lib/sync/elements.js index 26cdb503..e2e4ff21 100644 --- a/src/lib/sync/elements.js +++ b/src/lib/sync/elements.js @@ -6,6 +6,8 @@ import { get } from 'svelte/store'; axiosRetry(axios, { retries: 3 }); +const limit = 5000; + export const elementsSync = async () => { mapLoading.set('Checking local cache...'); // get elements from local @@ -18,39 +20,59 @@ export const elementsSync = async () => { let count = get(elementsSyncCount); elementsSyncCount.set(count + 1); - try { - mapLoading.set('Fetching elements...'); - const response = await axios.get('https://api.btcmap.org/v2/elements'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let elementsData = []; - if (response.data.length) { - mapLoading.set('Storing data...'); - // set response to local - localforage - .setItem('elements', response.data) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - mapLoading.set('Initial sync complete!'); - // set response to store - elements.set(response.data); - }) - .catch(function (err) { - mapLoading.set('Map loading complete!'); - elements.set(response.data); - elementError.set( - 'Could not store elements locally, please try again or contact BTC Map.' - ); - console.log(err); - }); - } else { + mapLoading.set('Fetching elements...'); + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/elements?updated_since=${updatedSince}&limit=${limit}` + ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + elementsData.filter( + (element) => !response.data.find((data) => data.id === element.id) + ); + response.data.forEach((data) => elementsData.push(data)); + } else { + elementError.set( + 'Elements API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { elementError.set( - 'Elements API returned an empty result, please try again or contact BTC Map.' + 'Could not load elements from API, please try again or contact BTC Map.' ); + console.log(error); + break; } - } catch (error) { - elementError.set( - 'Could not load elements from API, please try again or contact BTC Map.' - ); - console.log(error); + } while (responseCount === limit); + + if (elementsData.length) { + mapLoading.set('Storing data...'); + // set response to local + localforage + .setItem('elements', elementsData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + mapLoading.set('Initial sync complete!'); + // set response to store + elements.set(elementsData); + }) + .catch(function (err) { + mapLoading.set('Map loading complete!'); + elements.set(elementsData); + elementError.set( + 'Could not store elements locally, please try again or contact BTC Map.' + ); + console.log(err); + }); } } else { mapLoading.set('Local cache found!'); @@ -60,72 +82,90 @@ export const elementsSync = async () => { elementsSyncCount.set(count + 1); // start update sync from API - try { - // sort to get most recent record - let cacheSorted = [...value]; - cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + // sort to get most recent record + let cacheSorted = [...value]; + cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); - mapLoading.set('Fetching new elements...'); - const response = await axios.get( - `https://api.btcmap.org/v2/elements?updated_since=${cacheSorted[0]['updated_at']}` - ); + let updatedSince = cacheSorted[0]['updated_at']; + let responseCount; + let elementsData = value; + let useCachedData = false; - // update new records if they exist - let newElements = response.data; - - // check for new elements in local and purge if they exist - if (newElements.length) { - mapLoading.set('Storing data...'); - let updatedElements = value.filter((value) => { - if (newElements.find((element) => element.id === value.id)) { - return false; - } else { - return true; - } - }); + mapLoading.set('Fetching new elements...'); - // add new elements - updatedElements.forEach((element) => { - newElements.push(element); - }); + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/elements?updated_since=${updatedSince}&limit=${limit}` + ); + + // update new records if they exist + let newElements = response.data; - // set updated elements locally - localforage - .setItem('elements', newElements) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - mapLoading.set('Map loading complete!'); - // set updated elements to store - elements.set(newElements); - - // display data refresh icon on map - mapUpdates.set(true); - }) - .catch(function (err) { - // set updated elements to store - elements.set(newElements); - - // display data refresh icon on map - mapUpdates.set(true); - - elementError.set( - 'Could not update elements locally, please try again or contact BTC Map.' - ); - console.log(err); + // check for new elements in local and purge if they exist + if (newElements.length) { + updatedSince = newElements[newElements.length - 1]['updated_at']; + responseCount = newElements.length; + + elementsData.filter((value) => { + if (newElements.find((element) => element.id === value.id)) { + return false; + } else { + return true; + } }); - } else { - mapLoading.set('Map loading complete!'); + + // add new elements + newElements.forEach((element) => { + elementsData.push(element); + }); + } else { + mapLoading.set('Map loading complete!'); + // set cached elements to store + elements.set(value); + useCachedData = true; + break; + } + } catch (error) { // set cached elements to store elements.set(value); - } - } catch (error) { - // set cached elements to store - elements.set(value); + useCachedData = true; - elementError.set( - 'Could not update elements from API, please try again or contact BTC Map.' - ); - console.error(error); + elementError.set( + 'Could not update elements from API, please try again or contact BTC Map.' + ); + console.error(error); + break; + } + } while (responseCount === limit); + + if (!useCachedData) { + // set updated elements locally + mapLoading.set('Storing data...'); + + localforage + .setItem('elements', elementsData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + mapLoading.set('Map loading complete!'); + // set updated elements to store + elements.set(elementsData); + + // display data refresh icon on map + mapUpdates.set(true); + }) + .catch(function (err) { + // set updated elements to store + elements.set(elementsData); + + // display data refresh icon on map + mapUpdates.set(true); + + elementError.set( + 'Could not update elements locally, please try again or contact BTC Map.' + ); + console.log(err); + }); } } }) @@ -138,22 +178,42 @@ export const elementsSync = async () => { let count = get(elementsSyncCount); elementsSyncCount.set(count + 1); - try { - mapLoading.set('Fetching elements...'); - const response = await axios.get('https://api.btcmap.org/v2/elements'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let elementsData = []; - if (response.data.length) { - mapLoading.set('Initial sync complete!'); - // set response to store - elements.set(response.data); - } else { + mapLoading.set('Fetching elements...'); + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/elements?updated_since=${updatedSince}&limit=${limit}` + ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + elementsData.filter((element) => !response.data.find((data) => data.id === element.id)); + response.data.forEach((data) => elementsData.push(data)); + } else { + elementError.set( + 'Elements API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { elementError.set( - 'Elements API returned an empty result, please try again or contact BTC Map.' + 'Could not load elements from API, please try again or contact BTC Map.' ); + console.log(error); + break; } - } catch (error) { - elementError.set('Could not load elements from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (elementsData.length) { + mapLoading.set('Map loading complete!'); + // set response to store + elements.set(elementsData); } }); }; diff --git a/src/lib/sync/events.js b/src/lib/sync/events.js index ad3b4fae..c7abea40 100644 --- a/src/lib/sync/events.js +++ b/src/lib/sync/events.js @@ -5,6 +5,8 @@ import { events, eventError } from '$lib/store'; axiosRetry(axios, { retries: 3 }); +const limit = 50000; + export const eventsSync = async () => { // get events from local await localforage @@ -12,99 +14,134 @@ export const eventsSync = async () => { .then(async function (value) { // get events from API if initial sync if (!value) { - try { - const response = await axios.get('https://api.btcmap.org/v2/events'); - - if (response.data.length) { - // filter out deleted events - const eventsFiltered = response.data.filter((event) => !event['deleted_at']); - - // set response to local - localforage - .setItem('events', response.data) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set response to store - events.set(eventsFiltered); - }) - .catch(function (err) { - events.set(eventsFiltered); - eventError.set( - 'Could not store events locally, please try again or contact BTC Map.' - ); - console.log(err); - }); - } else { - eventError.set( - 'Events API returned an empty result, please try again or contact BTC Map.' + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let eventsData = []; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/events?updated_since=${updatedSince}&limit=${limit}` ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + eventsData.filter((event) => !response.data.find((data) => data.id === event.id)); + response.data.forEach((data) => eventsData.push(data)); + } else { + eventError.set( + 'Events API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + eventError.set('Could not load events from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - eventError.set('Could not load events from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (eventsData.length) { + // filter out deleted events + const eventsFiltered = eventsData.filter((event) => !event['deleted_at']); + + // set response to local + localforage + .setItem('events', eventsData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set response to store + events.set(eventsFiltered); + }) + .catch(function (err) { + events.set(eventsFiltered); + eventError.set( + 'Could not store events locally, please try again or contact BTC Map.' + ); + console.log(err); + }); } } else { // filter out deleted events const eventsFiltered = value.filter((event) => !event['deleted_at']); // start update sync from API - try { - // sort to get most recent record - let cacheSorted = [...value]; - cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + // sort to get most recent record + let cacheSorted = [...value]; + cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + + let updatedSince = cacheSorted[0]['updated_at']; + let responseCount; + let eventsData = value; + let useCachedData = false; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/events?updated_since=${updatedSince}&limit=${limit}` + ); - const response = await axios.get( - `https://api.btcmap.org/v2/events?updated_since=${cacheSorted[0]['updated_at']}` - ); + // update new records if they exist + let newEvents = response.data; - // update new records if they exist - let newEvents = response.data; - - // check for new events in local and purge if they exist - if (newEvents.length) { - let updatedEvents = value.filter((value) => { - if (newEvents.find((event) => event.id === value.id)) { - return false; - } else { - return true; - } - }); + // check for new events in local and purge if they exist + if (newEvents.length) { + updatedSince = newEvents[newEvents.length - 1]['updated_at']; + responseCount = newEvents.length; - // add new events - updatedEvents.forEach((event) => { - newEvents.push(event); - }); + eventsData.filter((value) => { + if (newEvents.find((event) => event.id === value.id)) { + return false; + } else { + return true; + } + }); - // filter out deleted events - const newEventsFiltered = newEvents.filter((event) => !event['deleted_at']); - - // set updated events locally - localforage - .setItem('events', newEvents) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set updated events to store - events.set(newEventsFiltered); - }) - .catch(function (err) { - // set updated events to store - events.set(newEventsFiltered); - - eventError.set( - 'Could not update events locally, please try again or contact BTC Map.' - ); - console.log(err); + // add new events + newEvents.forEach((event) => { + eventsData.push(event); }); - } else { + } else { + // load events from cache + events.set(eventsFiltered); + useCachedData = true; + break; + } + } catch (error) { // load events from cache events.set(eventsFiltered); + useCachedData = true; + + eventError.set( + 'Could not update events from API, please try again or contact BTC Map.' + ); + console.error(error); + break; } - } catch (error) { - // load events from cache - events.set(eventsFiltered); + } while (responseCount === limit); - eventError.set('Could not update events from API, please try again or contact BTC Map.'); - console.error(error); + if (!useCachedData) { + // filter out deleted events + const newEventsFiltered = eventsData.filter((event) => !event['deleted_at']); + + // set updated events locally + localforage + .setItem('events', eventsData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set updated events to store + events.set(newEventsFiltered); + }) + .catch(function (err) { + // set updated events to store + events.set(newEventsFiltered); + + eventError.set( + 'Could not update events locally, please try again or contact BTC Map.' + ); + console.log(err); + }); } } }) @@ -113,23 +150,40 @@ export const eventsSync = async () => { eventError.set('Could not load events locally, please try again or contact BTC Map.'); console.log(err); - try { - const response = await axios.get('https://api.btcmap.org/v2/events'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let eventsData = []; - if (response.data.length) { - // filter out deleted events - const eventsFiltered = response.data.filter((event) => !event['deleted_at']); - - // set response to store - events.set(eventsFiltered); - } else { - eventError.set( - 'Events API returned an empty result, please try again or contact BTC Map.' + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/events?updated_since=${updatedSince}&limit=${limit}` ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + eventsData.filter((event) => !response.data.find((data) => data.id === event.id)); + response.data.forEach((data) => eventsData.push(data)); + } else { + eventError.set( + 'Events API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + eventError.set('Could not load events from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - eventError.set('Could not load events from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (eventsData.length) { + // filter out deleted events + const eventsFiltered = eventsData.filter((event) => !event['deleted_at']); + + // set response to store + events.set(eventsFiltered); } }); }; diff --git a/src/lib/sync/reports.js b/src/lib/sync/reports.js index 0c62cbc6..9d66899b 100644 --- a/src/lib/sync/reports.js +++ b/src/lib/sync/reports.js @@ -5,6 +5,8 @@ import { reports, reportError } from '$lib/store'; axiosRetry(axios, { retries: 3 }); +const limit = 20000; + export const reportsSync = async () => { // get reports from local await localforage @@ -12,101 +14,136 @@ export const reportsSync = async () => { .then(async function (value) { // get reports from API if initial sync if (!value) { - try { - const response = await axios.get('https://api.btcmap.org/v2/reports'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let reportsData = []; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&limit=${limit}` + ); - if (response.data.length) { - // filter out deleted reports - const reportsFiltered = response.data.filter((report) => !report['deleted_at']); - - // set response to local - localforage - .setItem('reports', response.data) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set response to store - reports.set(reportsFiltered); - }) - .catch(function (err) { - reports.set(reportsFiltered); - reportError.set( - 'Could not store reports locally, please try again or contact BTC Map.' - ); - console.log(err); - }); - } else { + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + reportsData.filter((report) => !response.data.find((data) => data.id === report.id)); + response.data.forEach((data) => reportsData.push(data)); + } else { + reportError.set( + 'Reports API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { reportError.set( - 'Reports API returned an empty result, please try again or contact BTC Map.' + 'Could not load reports from API, please try again or contact BTC Map.' ); + console.log(error); + break; } - } catch (error) { - reportError.set('Could not load reports from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (reportsData.length) { + // filter out deleted reports + const reportsFiltered = reportsData.filter((report) => !report['deleted_at']); + + // set response to local + localforage + .setItem('reports', reportsData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set response to store + reports.set(reportsFiltered); + }) + .catch(function (err) { + reports.set(reportsFiltered); + reportError.set( + 'Could not store reports locally, please try again or contact BTC Map.' + ); + console.log(err); + }); } } else { // filter out deleted reports const reportsFiltered = value.filter((report) => !report['deleted_at']); // start update sync from API - try { - // sort to get most recent record - let cacheSorted = [...value]; - cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + // sort to get most recent record + let cacheSorted = [...value]; + cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + + let updatedSince = cacheSorted[0]['updated_at']; + let responseCount; + let reportsData = value; + let useCachedData = false; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&limit=${limit}` + ); - const response = await axios.get( - `https://api.btcmap.org/v2/reports?updated_since=${cacheSorted[0]['updated_at']}` - ); + // update new records if they exist + let newReports = response.data; - // update new records if they exist - let newReports = response.data; - - // check for new reports in local and purge if they exist - if (newReports.length) { - let updatedReports = value.filter((value) => { - if (newReports.find((report) => report.id === value.id)) { - return false; - } else { - return true; - } - }); + // check for new reports in local and purge if they exist + if (newReports.length) { + updatedSince = newReports[newReports.length - 1]['updated_at']; + responseCount = newReports.length; - // add new reports - updatedReports.forEach((report) => { - newReports.push(report); - }); + reportsData.filter((value) => { + if (newReports.find((report) => report.id === value.id)) { + return false; + } else { + return true; + } + }); - // filter out deleted reports - const newReportsFiltered = newReports.filter((report) => !report['deleted_at']); - - // set updated reports locally - localforage - .setItem('reports', newReports) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set updated reports to store - reports.set(newReportsFiltered); - }) - .catch(function (err) { - // set updated reports to store - reports.set(newReportsFiltered); - - reportError.set( - 'Could not update reports locally, please try again or contact BTC Map.' - ); - console.log(err); + // add new reports + newReports.forEach((report) => { + reportsData.push(report); }); - } else { + } else { + // load reports from cache + reports.set(reportsFiltered); + useCachedData = true; + break; + } + } catch (error) { // load reports from cache reports.set(reportsFiltered); + useCachedData = true; + + reportError.set( + 'Could not update reports from API, please try again or contact BTC Map.' + ); + console.error(error); + break; } - } catch (error) { - // load reports from cache - reports.set(reportsFiltered); + } while (responseCount === limit); - reportError.set( - 'Could not update reports from API, please try again or contact BTC Map.' - ); - console.error(error); + if (!useCachedData) { + // filter out deleted reports + const newReportsFiltered = reportsData.filter((report) => !report['deleted_at']); + + // set updated reports locally + localforage + .setItem('reports', reportsData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set updated reports to store + reports.set(newReportsFiltered); + }) + .catch(function (err) { + // set updated reports to store + reports.set(newReportsFiltered); + + reportError.set( + 'Could not update reports locally, please try again or contact BTC Map.' + ); + console.log(err); + }); } } }) @@ -115,23 +152,40 @@ export const reportsSync = async () => { reportError.set('Could not load reports locally, please try again or contact BTC Map.'); console.log(err); - try { - const response = await axios.get('https://api.btcmap.org/v2/reports'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let reportsData = []; - if (response.data.length) { - // filter out deleted reports - const reportsFiltered = response.data.filter((report) => !report['deleted_at']); - - // set response to store - reports.set(reportsFiltered); - } else { - reportError.set( - 'Reports API returned an empty result, please try again or contact BTC Map.' + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/reports?updated_since=${updatedSince}&limit=${limit}` ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + reportsData.filter((report) => !response.data.find((data) => data.id === report.id)); + response.data.forEach((data) => reportsData.push(data)); + } else { + reportError.set( + 'Reports API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + reportError.set('Could not load reports from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - reportError.set('Could not load reports from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (reportsData.length) { + // filter out deleted reports + const reportsFiltered = reportsData.filter((report) => !report['deleted_at']); + + // set response to store + reports.set(reportsFiltered); } }); }; diff --git a/src/lib/sync/users.js b/src/lib/sync/users.js index 36ea80d6..6bf718cd 100644 --- a/src/lib/sync/users.js +++ b/src/lib/sync/users.js @@ -5,6 +5,8 @@ import { users, userError } from '$lib/store'; axiosRetry(axios, { retries: 3 }); +const limit = 7500; + export const usersSync = async () => { // clear potentially broken users v1 sync due to top level ID changing from string to int await localforage @@ -33,99 +35,128 @@ export const usersSync = async () => { .then(async function (value) { // get users from API if initial sync if (!value) { - try { - const response = await axios.get('https://api.btcmap.org/v2/users'); - - if (response.data.length) { - // filter out deleted users - const usersFiltered = response.data.filter((user) => !user['deleted_at']); - - // set response to local - localforage - .setItem('users_v2', response.data) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set response to store - users.set(usersFiltered); - }) - .catch(function (err) { - users.set(usersFiltered); - userError.set( - 'Could not store users locally, please try again or contact BTC Map.' - ); - console.log(err); - }); - } else { - userError.set( - 'Users API returned an empty result, please try again or contact BTC Map.' + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let usersData = []; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/users?updated_since=${updatedSince}&limit=${limit}` ); + + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + usersData.filter((user) => !response.data.find((data) => data.id === user.id)); + response.data.forEach((data) => usersData.push(data)); + } else { + userError.set( + 'Users API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + userError.set('Could not load users from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - userError.set('Could not load users from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (usersData.length) { + // filter out deleted users + const usersFiltered = usersData.filter((user) => !user['deleted_at']); + + // set response to local + localforage + .setItem('users_v2', usersData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set response to store + users.set(usersFiltered); + }) + .catch(function (err) { + users.set(usersFiltered); + userError.set('Could not store users locally, please try again or contact BTC Map.'); + console.log(err); + }); } } else { // filter out deleted users const usersFiltered = value.filter((user) => !user['deleted_at']); // start update sync from API - try { - // sort to get most recent record - let cacheSorted = [...value]; - cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + // sort to get most recent record + let cacheSorted = [...value]; + cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at'])); + + let updatedSince = cacheSorted[0]['updated_at']; + let responseCount; + let usersData = value; + let useCachedData = false; + + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/users?updated_since=${updatedSince}&limit=${limit}` + ); - const response = await axios.get( - `https://api.btcmap.org/v2/users?updated_since=${cacheSorted[0]['updated_at']}` - ); + // update new records if they exist + let newUsers = response.data; - // update new records if they exist - let newUsers = response.data; - - // check for new users in local and purge if they exist - if (newUsers.length) { - let updatedUsers = value.filter((value) => { - if (newUsers.find((user) => user.id == value.id)) { - return false; - } else { - return true; - } - }); + // check for new users in local and purge if they exist + if (newUsers.length) { + updatedSince = newUsers[newUsers.length - 1]['updated_at']; + responseCount = newUsers.length; - // add new users - updatedUsers.forEach((user) => { - newUsers.push(user); - }); + usersData.filter((value) => { + if (newUsers.find((user) => user.id == value.id)) { + return false; + } else { + return true; + } + }); - // filter out deleted users - const newUsersFiltered = newUsers.filter((user) => !user['deleted_at']); - - // set updated users locally - localforage - .setItem('users_v2', newUsers) - // eslint-disable-next-line no-unused-vars - .then(function (value) { - // set updated users to store - users.set(newUsersFiltered); - }) - .catch(function (err) { - // set updated users to store - users.set(newUsersFiltered); - - userError.set( - 'Could not update users locally, please try again or contact BTC Map.' - ); - console.log(err); + // add new users + newUsers.forEach((user) => { + usersData.push(user); }); - } else { + } else { + // load users from cache + users.set(usersFiltered); + useCachedData = true; + break; + } + } catch (error) { // load users from cache users.set(usersFiltered); + useCachedData = true; + + userError.set('Could not update users from API, please try again or contact BTC Map.'); + console.error(error); + break; } - } catch (error) { - // load users from cache - users.set(usersFiltered); + } while (responseCount === limit); - userError.set('Could not update users from API, please try again or contact BTC Map.'); - console.error(error); + if (!useCachedData) { + // filter out deleted users + const newUsersFiltered = usersData.filter((user) => !user['deleted_at']); + + // set updated users locally + localforage + .setItem('users_v2', usersData) + // eslint-disable-next-line no-unused-vars + .then(function (value) { + // set updated users to store + users.set(newUsersFiltered); + }) + .catch(function (err) { + // set updated users to store + users.set(newUsersFiltered); + + userError.set('Could not update users locally, please try again or contact BTC Map.'); + console.log(err); + }); } } }) @@ -134,21 +165,40 @@ export const usersSync = async () => { userError.set('Could not load users locally, please try again or contact BTC Map.'); console.log(err); - try { - const response = await axios.get('https://api.btcmap.org/v2/users'); + let updatedSince = '2022-01-01T00:00:00.000Z'; + let responseCount; + let usersData = []; - if (response.data.length) { - // filter out deleted users - const usersFiltered = response.data.filter((user) => !user['deleted_at']); + do { + try { + const response = await axios.get( + `https://api.btcmap.org/v2/users?updated_since=${updatedSince}&limit=${limit}` + ); - // set response to store - users.set(usersFiltered); - } else { - userError.set('Users API returned an empty result, please try again or contact BTC Map.'); + if (response.data.length) { + updatedSince = response.data[response.data.length - 1]['updated_at']; + responseCount = response.data.length; + usersData.filter((user) => !response.data.find((data) => data.id === user.id)); + response.data.forEach((data) => usersData.push(data)); + } else { + userError.set( + 'Users API returned an empty result, please try again or contact BTC Map.' + ); + break; + } + } catch (error) { + userError.set('Could not load users from API, please try again or contact BTC Map.'); + console.log(error); + break; } - } catch (error) { - userError.set('Could not load users from API, please try again or contact BTC Map.'); - console.log(error); + } while (responseCount === limit); + + if (usersData.length) { + // filter out deleted users + const usersFiltered = usersData.filter((user) => !user['deleted_at']); + + // set response to store + users.set(usersFiltered); } }); };