Skip to content

Commit

Permalink
refactor sync to use incremental limit
Browse files Browse the repository at this point in the history
  • Loading branch information
@secondl1ght
secondl1ght committed Nov 26, 2023
1 parent 9f5bb6e commit 98c8139
Show file tree
Hide file tree
Showing 5 changed files with 723 additions and 455 deletions.
226 changes: 138 additions & 88 deletions src/lib/sync/areas.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,106 +5,137 @@ import { areas, areaError } from '$lib/store';

axiosRetry(axios, { retries: 3 });

const limit = 500;

export const areasSync = async () => {
// get areas from local
await localforage
.getItem('areas')
.then(async function (value) {
// get areas from API if initial sync
if (!value) {
try {
const response = await axios.get('https://api.btcmap.org/v2/areas');

if (response.data.length) {
// filter out deleted areas
const areasFiltered = response.data.filter((area) => !area['deleted_at']);

// set response to local
localforage
.setItem('areas', response.data)
// eslint-disable-next-line no-unused-vars
.then(function (value) {
// set response to store
areas.set(areasFiltered);
})
.catch(function (err) {
areas.set(areasFiltered);
areaError.set(
'Could not store areas locally, please try again or contact BTC Map.'
);
console.log(err);
});
} else {
areaError.set(
'Areas API returned an empty result, please try again or contact BTC Map.'
let updatedSince = '2022-01-01T00:00:00.000Z';
let responseCount;
let areasData = [];

do {
try {
const response = await axios.get(
`https://api.btcmap.org/v2/areas?updated_since=${updatedSince}&limit=${limit}`
);

if (response.data.length) {
updatedSince = response.data[response.data.length - 1]['updated_at'];
responseCount = response.data.length;
areasData.filter((area) => !response.data.find((data) => data.id === area.id));
response.data.forEach((data) => areasData.push(data));
} else {
areaError.set(
'Areas API returned an empty result, please try again or contact BTC Map.'
);
break;
}
} catch (error) {
areaError.set('Could not load areas from API, please try again or contact BTC Map.');
console.log(error);
break;
}
} catch (error) {
areaError.set('Could not load areas from API, please try again or contact BTC Map.');
console.log(error);
} while (responseCount === limit);

if (areasData.length) {
// filter out deleted areas
const areasFiltered = areasData.filter((area) => !area['deleted_at']);

// set response to local
localforage
.setItem('areas', areasData)
// eslint-disable-next-line no-unused-vars
.then(function (value) {
// set response to store
areas.set(areasFiltered);
})
.catch(function (err) {
areas.set(areasFiltered);
areaError.set('Could not store areas locally, please try again or contact BTC Map.');
console.log(err);
});
}
} else {
// filter out deleted areas
const areasFiltered = value.filter((area) => !area['deleted_at']);

// start update sync from API
try {
// sort to get most recent record
let cacheSorted = [...value];
cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at']));
// sort to get most recent record
let cacheSorted = [...value];
cacheSorted.sort((a, b) => Date.parse(b['updated_at']) - Date.parse(a['updated_at']));

let updatedSince = cacheSorted[0]['updated_at'];
let responseCount;
let areasData = value;
let useCachedData = false;

do {
try {
const response = await axios.get(
`https://api.btcmap.org/v2/areas?updated_since=${updatedSince}&limit=${limit}`
);

const response = await axios.get(
`https://api.btcmap.org/v2/areas?updated_since=${cacheSorted[0]['updated_at']}`
);
// update new records if they exist
let newAreas = response.data;

// update new records if they exist
let newAreas = response.data;

// check for new areas in local and purge if they exist
if (newAreas.length) {
let updatedAreas = value.filter((value) => {
if (newAreas.find((area) => area.id === value.id)) {
return false;
} else {
return true;
}
});
// check for new areas in local and purge if they exist
if (newAreas.length) {
updatedSince = newAreas[newAreas.length - 1]['updated_at'];
responseCount = newAreas.length;

// add new areas
updatedAreas.forEach((area) => {
newAreas.push(area);
});
areasData.filter((value) => {
if (newAreas.find((area) => area.id === value.id)) {
return false;
} else {
return true;
}
});

// filter out deleted areas
const newAreasFiltered = newAreas.filter((area) => !area['deleted_at']);

// set updated areas locally
localforage
.setItem('areas', newAreas)
// eslint-disable-next-line no-unused-vars
.then(function (value) {
// set updated areas to store
areas.set(newAreasFiltered);
})
.catch(function (err) {
// set updated areas to store
areas.set(newAreasFiltered);

areaError.set(
'Could not update areas locally, please try again or contact BTC Map.'
);
console.log(err);
// add new areas
newAreas.forEach((area) => {
areasData.push(area);
});
} else {
} else {
// load areas from cache
areas.set(areasFiltered);
useCachedData = true;
break;
}
} catch (error) {
// load areas from cache
areas.set(areasFiltered);
useCachedData = true;

areaError.set('Could not update areas from API, please try again or contact BTC Map.');
console.error(error);
break;
}
} catch (error) {
// load areas from cache
areas.set(areasFiltered);
} while (responseCount === limit);

areaError.set('Could not update areas from API, please try again or contact BTC Map.');
console.error(error);
if (!useCachedData) {
// filter out deleted areas
const newAreasFiltered = areasData.filter((area) => !area['deleted_at']);

// set updated areas locally
localforage
.setItem('areas', areasData)
// eslint-disable-next-line no-unused-vars
.then(function (value) {
// set updated areas to store
areas.set(newAreasFiltered);
})
.catch(function (err) {
// set updated areas to store
areas.set(newAreasFiltered);

areaError.set('Could not update areas locally, please try again or contact BTC Map.');
console.log(err);
});
}
}
})
Expand All @@ -113,21 +144,40 @@ export const areasSync = async () => {
areaError.set('Could not load areas locally, please try again or contact BTC Map.');
console.log(err);

try {
const response = await axios.get('https://api.btcmap.org/v2/areas');
let updatedSince = '2022-01-01T00:00:00.000Z';
let responseCount;
let areasData = [];

if (response.data.length) {
// filter out deleted areas
const areasFiltered = response.data.filter((area) => !area['deleted_at']);
do {
try {
const response = await axios.get(
`https://api.btcmap.org/v2/areas?updated_since=${updatedSince}&limit=${limit}`
);

// set response to store
areas.set(areasFiltered);
} else {
areaError.set('Areas API returned an empty result, please try again or contact BTC Map.');
if (response.data.length) {
updatedSince = response.data[response.data.length - 1]['updated_at'];
responseCount = response.data.length;
areasData.filter((area) => !response.data.find((data) => data.id === area.id));
response.data.forEach((data) => areasData.push(data));
} else {
areaError.set(
'Areas API returned an empty result, please try again or contact BTC Map.'
);
break;
}
} catch (error) {
areaError.set('Could not load areas from API, please try again or contact BTC Map.');
console.log(error);
break;
}
} catch (error) {
areaError.set('Could not load areas from API, please try again or contact BTC Map.');
console.log(error);
} while (responseCount === limit);

if (areasData.length) {
// filter out deleted areas
const areasFiltered = areasData.filter((area) => !area['deleted_at']);

// set response to store
areas.set(areasFiltered);
}
});
};
Loading

0 comments on commit 98c8139

Please sign in to comment.