fix(referer): add referer header to cURL requests (#483)

pelias · Mar 9, 2021 · 3cc268c · 3cc268c
1 parent 863b528
commit 3cc268c
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/utils/download_all.js b/utils/download_all.js
@@ -34,6 +34,7 @@ function downloadAll(config, callback) {
 function downloadBundle(targetDir, config, sourceUrl, callback) {
 
   const tmpZipFile = temp.path({suffix: '.zip'});
+  const referer = config.get('imports.openaddresses.dataReferer') || 'https://pelias-results.openaddresses.io';
 
   async.series(
     [
@@ -44,7 +45,7 @@ function downloadBundle(targetDir, config, sourceUrl, callback) {
           const s3Options = config.imports.openaddresses.s3Options || '';
           child_process.exec(`aws s3 cp ${sourceUrl} ${tmpZipFile} ${s3Options}`, callback);
         } else {
-          child_process.exec(`curl -s -L -X GET -o ${tmpZipFile} ${sourceUrl}`, callback);
+          child_process.exec(`curl -s -L -X GET --referer ${referer} -o ${tmpZipFile} ${sourceUrl}`, callback);
         }
       },
       // unzip file into target directory

diff --git a/utils/download_filtered.js b/utils/download_filtered.js
@@ -68,14 +68,15 @@ function getFiles(config, targetDir, main_callback){
 
 function downloadSource(targetDir, file, main_callback) {
   const errorsFatal = config.get('imports.openaddresses.missingFilesAreFatal');
+  const referer = config.get('imports.openaddresses.dataReferer') || 'https://pelias-results.openaddresses.io';
   logger.info(`Downloading ${file.csv}`);
 
   async.series(
     [
       // download the zip file into the temp directory
       (callback) => {
         logger.debug(`downloading ${file.url}`);
-        child_process.exec(`curl -s -L -X GET -o ${file.zip} ${file.url}`, callback);
+        child_process.exec(`curl -s -L -X GET --referer ${referer} -o ${file.zip} ${file.url}`, callback);
       },
       // unzip file into target directory
       (callback) => {