diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 59293db..6b8357d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,29 +1,36 @@ -name: Deploy to Cloud Run - +name: Default test on: push: - branches: - - main + branches-ignore: + - 'master' + - 'staging' jobs: - deploy: - name: Deploy to Cloud Run + test-build: runs-on: ubuntu-latest + strategy: + matrix: + node-version: [22.x] + steps: - # Step 1: Checkout the repository - - name: Checkout code - uses: actions/checkout@v3 - - # Step 2: Set up Google Cloud SDK - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} # Set this secret in your GitHub repo - service_account_key: ${{ secrets.GCP_SA_KEY }} # Set this secret in your GitHub repo - export_default_credentials: true - - # Step 3: Trigger Cloud Build - - name: Trigger Cloud Build - run: | - gcloud builds submit --config cloudbuild.yaml --substitutions=_PROJECT_ID=${{ secrets.GCP_PROJECT_ID }} + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + - name: npm install + run: | + npm install + - name: npm test + run: | + npm test + + - name: npm test:e2e + run: | + npm run test:e2e + + - name: npm build + run: | + npm run build --if-present \ No newline at end of file diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml new file mode 100644 index 0000000..d7153cf --- /dev/null +++ b/.github/workflows/production.yml @@ -0,0 +1,76 @@ +name: Deploy Production to Google Cloud Run +on: + workflow_dispatch: + inputs: + trigger: + description: 'Production Deploy' + required: true + default: 'build and deploy' + +jobs: + test-build: + name: Production - ${{ github.event.head_commit.message }} + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [21.x] + + steps: + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + - name: npm install + run: | + npm install + - name: npm test + run: | + npm test + - name: npm test:e2e + run: | + npm run test:e2e + + - name: npm build + run: | + npm run build --if-present + + + setup-build-publish-deploy: + needs: [test-build] + name: Setup, Build, Publish, and Deploy + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@master + + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + credentials_json: '${{ secrets.GCLOUD_SERVICE_KEY }}' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - name: 'Use gcloud CLI' + run: 'gcloud info' + + - run: | + # Set up docker to authenticate + # via gcloud command-line tool. + gcloud auth configure-docker + + # Build the Docker image + - name: Build + run: | + docker build . --tag gcr.io/${{ secrets.GCLOUD_PROJECT_ID }}/${{ secrets.GCLOUD_APP_NAME }} + # Push the Docker image to Google Container Registry + - name: Publish + run: | + docker push gcr.io/${{ secrets.GCLOUD_PROJECT_ID }}/${{ secrets.GCLOUD_APP_NAME }} + + # Deploy the Docker image to the GKE cluster + - name: Deploy + run: | + gcloud components install beta && gcloud beta run deploy ${{ secrets.GCLOUD_APP_NAME }} --image gcr.io/${{ secrets.GCLOUD_PROJECT_ID }}/${{ secrets.GCLOUD_APP_NAME }}:latest --project ${{ secrets.GCLOUD_PROJECT_ID }} --region us-central1 --allow-unauthenticated --platform managed diff --git a/extra-datasets/brands/brand-to-wikidata.json b/extra-datasets/brands/brand-to-wikidata.json index aa4c5bd..c1a170e 100644 --- a/extra-datasets/brands/brand-to-wikidata.json +++ b/extra-datasets/brands/brand-to-wikidata.json @@ -1,6 +1,12 @@ [ - {"brand": "Woolworths Metro","wikidata":"Q111772555", "operator": "Woolworths Group", "brand:wikidata": "Q607272"}, - {"brand": "Woolworths","wikidata":"Q3249145", "operator": "Woolworths Group", "brand:wikidata": "Q607272"}, - {"brand": "Coles","wikidata":"Q1108172", "operator": "Coles Group", "brand:wikidata": "Q1339055"}, - {"brand": "Coles Express","wikidata":"Q5144653", "operator": "Coles Group", "brand:wikidata": "Q1339055"} + {"brand": "Woolworths Metro","wikidata":"Q111772555", "operator": "Woolworths Group", "brand:wikidata": "Q607272", "countries":["AU"]}, + {"brand": "Woolworths","wikidata":"Q3249145", "operator": "Woolworths Group", "brand:wikidata": "Q607272", "countries":["AU"]}, + {"brand": "Coles","wikidata":"Q1108172", "operator": "Coles Group", "brand:wikidata": "Q1339055", "countries":["AU"]}, + {"brand": "Coles Express","wikidata":"Q5144653", "operator": "Coles Group", "brand:wikidata": "Q1339055", "countries":["AU"]}, + {"brand":"Australia Post","wikidata":"Q1142936", "countries":["AU"]}, + {"brand":"TAB","wikidata":"Q110288149", "countries":["AU"]}, + {"brand":"BWS","wikidata":"Q4836848", "countries":["AU"]}, + {"brand":"BP","wikidata":"Q50736918", "countries":["AU"], "operator": "BP", "brand:wikidata": "Q152057" }, + {"brand":"Subway Australia","wikidata":"Q244457" , "countries":["AU"], "operator": "Subway", "brand:wikidata": "Q244457"}, + {"brand":"McDonald's","wikidata":"Q82813727", "countries":["AU"], "operator": "McDonald’s", "brand:wikidata": "Q38076" } ] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 890f349..980d180 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,8 @@ "class-validator": "^0.14.1", "dotenv": "^16.4.5", "reflect-metadata": "^0.2.0", - "rxjs": "^7.8.1" + "rxjs": "^7.8.1", + "theauthapi": "^1.0.1-2.1" }, "devDependencies": { "@nestjs/cli": "^10.0.0", @@ -757,6 +758,18 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.25.7", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.7.tgz", @@ -3289,6 +3302,19 @@ "dev": true, "license": "MIT" }, + "node_modules/assert": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/assert/-/assert-2.1.0.tgz", + "integrity": "sha512-eLHpSK/Y4nhMJ07gDaAzoX/XAKS8PSaojml3M0DM4JpV1LAi5JOJ/p6H/XWrl8L+DzVEvVCW1z3vWAaB9oTsQw==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.2", + "is-nan": "^1.3.2", + "object-is": "^1.1.5", + "object.assign": "^4.1.4", + "util": "^0.12.5" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -3311,6 +3337,40 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/axios": { + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.14.0" + } + }, + "node_modules/axios-retry": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.9.1.tgz", + "integrity": "sha512-8PJDLJv7qTTMMwdnbMvrLYuvB47M81wRtxQmEdV5w4rgbTXTt+vtPkXwajOfOdSyv/wZICJOC+/UhXH4aQ/R+w==", + "license": "Apache-2.0", + "dependencies": { + "@babel/runtime": "^7.15.4", + "is-retry-allowed": "^2.2.0" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -4296,6 +4356,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -5284,6 +5361,35 @@ "dev": true, "license": "ISC" }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.1.3" + } + }, "node_modules/foreground-child": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", @@ -5727,6 +5833,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -5990,6 +6111,22 @@ "node": "*" } }, + "node_modules/is-arguments": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", + "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -6010,6 +6147,18 @@ "node": ">=8" } }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-core-module": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", @@ -6056,6 +6205,21 @@ "node": ">=6" } }, + "node_modules/is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "license": "MIT", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -6079,6 +6243,22 @@ "node": ">=8" } }, + "node_modules/is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -6089,6 +6269,18 @@ "node": ">=0.12.0" } }, + "node_modules/is-retry-allowed": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -6101,6 +6293,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-typed-array": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", + "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.14" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", @@ -7608,6 +7815,49 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/object-is": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.6.tgz", + "integrity": "sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.7", + "define-properties": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.assign": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", + "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.5", + "define-properties": "^1.2.1", + "has-symbols": "^1.0.3", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -7972,6 +8222,15 @@ "node": ">=4" } }, + "node_modules/possible-typed-array-names": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", + "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -8229,6 +8488,18 @@ "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", "license": "Apache-2.0" }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, + "node_modules/remove-trailing-slash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz", + "integrity": "sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA==", + "license": "MIT" + }, "node_modules/repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", @@ -9226,6 +9497,18 @@ "dev": true, "license": "MIT" }, + "node_modules/theauthapi": { + "version": "1.0.1-2.1", + "resolved": "https://registry.npmjs.org/theauthapi/-/theauthapi-1.0.1-2.1.tgz", + "integrity": "sha512-VOwnVLsPqDGySDZ2G58HfDb2VvsSN4kG8OZNl8bU8I3IEUuSWd3C0mL5OVUjNm2XNphu88boHHsTTZ5eyb7IdQ==", + "license": "MIT", + "dependencies": { + "assert": "^2.0.0", + "axios": "^0.21.4", + "axios-retry": "^3.1.9", + "remove-trailing-slash": "^0.1.1" + } + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -9621,6 +9904,19 @@ "punycode": "^2.1.0" } }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -9845,6 +10141,25 @@ "node": ">= 8" } }, + "node_modules/which-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", + "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.7", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/package.json b/package.json index d8934b0..97bfd43 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "class-validator": "^0.14.1", "dotenv": "^16.4.5", "reflect-metadata": "^0.2.0", - "rxjs": "^7.8.1" + "rxjs": "^7.8.1", + "theauthapi": "^1.0.1-2.1" }, "devDependencies": { "@nestjs/cli": "^10.0.0", diff --git a/src/app.controller.spec.ts b/src/app.controller.spec.ts index d22f389..0e3e86e 100644 --- a/src/app.controller.spec.ts +++ b/src/app.controller.spec.ts @@ -15,8 +15,8 @@ describe('AppController', () => { }); describe('root', () => { - it('should return "Hello World!"', () => { - expect(appController.getHello()).toBe('Hello World!'); + it('should return hello', () => { + expect(appController.getHello()).toBe('API by ThatAPICompany.com, Data by OvertureMaps.org'); }); }); }); diff --git a/src/app.module.ts b/src/app.module.ts index f93e423..156ba63 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,13 +1,35 @@ // src/app.module.ts -import { Module } from '@nestjs/common'; +import { Module, NestMiddleware, MiddlewareConsumer, Logger, RequestMethod } from '@nestjs/common'; import { PlacesController } from './places/places.controller'; import { BigQueryService } from './bigquery/bigquery.service'; import { GcsService } from './gcs/gcs.service'; import { ConfigModule } from '@nestjs/config'; +import {Request, Response} from 'express' +import { AuthAPIMiddleware } from './middleware/auth-api.middleware'; +import { AppController } from './app.controller'; +import { AppService } from './app.service'; @Module({ imports: [ConfigModule.forRoot()], - controllers: [PlacesController], - providers: [BigQueryService, GcsService], + controllers: [AppController,PlacesController], + providers: [BigQueryService, GcsService,AppService], }) -export class AppModule {} +export class AppModule { +configure(consumer: MiddlewareConsumer) { + consumer + .apply(LoggerMiddleware) + .forRoutes('*'); + + consumer.apply(AuthAPIMiddleware) + .forRoutes('*'); + + } +} + +class LoggerMiddleware implements NestMiddleware { + use(req:Request, res:Response, next: Function) { + + Logger.debug(`Request ${req.method} ${req.originalUrl}`) + next(); + } + } \ No newline at end of file diff --git a/src/app.service.ts b/src/app.service.ts index 927d7cc..216d3dc 100644 --- a/src/app.service.ts +++ b/src/app.service.ts @@ -3,6 +3,6 @@ import { Injectable } from '@nestjs/common'; @Injectable() export class AppService { getHello(): string { - return 'Hello World!'; + return 'API by ThatAPICompany.com, Data by OvertureMaps.org'; } } diff --git a/src/bigquery/bigquery.service.ts b/src/bigquery/bigquery.service.ts index 1f29448..40e16cc 100644 --- a/src/bigquery/bigquery.service.ts +++ b/src/bigquery/bigquery.service.ts @@ -3,6 +3,14 @@ import { Injectable, Logger } from '@nestjs/common'; import { BigQuery } from '@google-cloud/bigquery'; import { Place } from '../places/interfaces/place.interface'; +interface IQueryStatistics { + totalBytesProcessed: number; + totalBytesBilled: number; + billedAmountInGB: number; + bytesProcessedInGB: number; + durationMs: number; +} + @Injectable() export class BigQueryService { private bigQueryClient: BigQuery; @@ -92,11 +100,12 @@ export class BigQueryService { latitude?: number, longitude?: number, radius: number = 1000, + categories?: string[], minimum_places: number = 10, require_wikidata: boolean = false - ): Promise<{ brand: string; wikidata: string }[]> { - let query = ` + ): Promise<{ brand: string; wikidata: string; counts:{ places:number} }[]> { + let query = `-- Overture Maps API: Get brands nearby SELECT DISTINCT brand.names.primary AS brand, brand.wikidata AS wikidata, count(id) as count_places FROM \`bigquery-public-data.overture_maps.place\` `; @@ -110,6 +119,9 @@ export class BigQueryService { ) <= ${radius}`; } query += ` AND brand IS NOT NULL`; + if(categories && categories.length > 0){ + query += ` AND category.primary IN UNNEST(["${categories.join('","')}"])`; + } if (require_wikidata) { query += ` AND brand.wikidata IS NOT NULL`; } @@ -129,64 +141,149 @@ export class BigQueryService { return rows.map((row: any) => ({ brand: row.brand, wikidata: row.wikidata, - count_places: row.count_places, + counts:{ + places: row.count_places + } })); } - async getPlaceCountsByCountry(): Promise<{ country: string; count_places: number }[]> { - const query = ` - SELECT addresses.list[OFFSET(0)].element.country AS country, COUNT(id) AS count_places + async getPlaceCountsByCountry(): Promise<{ country: string; counts:{ places:number, brands:number} }[]> { + const query = `-- Overture Maps API: Get place counts by country + SELECT addresses.list[OFFSET(0)].element.country AS country, COUNT(id) AS count_places, count(DISTINCT brand.names.primary ) as count_brands FROM \`bigquery-public-data.overture_maps.place\` GROUP BY country ORDER BY count_places DESC; `; - const options = { - query: query, - location: 'US', // Adjust the location if necessary - }; - - const [rows] = await this.bigQueryClient.query(options); + const {rows} = await this.runQuery(query); return rows.map((row: any) => ({ country: row.country, - count_places: row.count_places, + counts:{ + places: row.count_places, + brands: row.count_brands + } })); } - async getPlacesNearby( latitude: number, longitude: number, radius: number = 1000, - wikidata?: string, - country?: string + brand_wikidata?: string, + brand_name?: string, + country?: string, + categories?: string[], + min_confidence?: number, + limit?: number ): Promise { - // Build the query with optional filters for wikidata and country - let query = ` - SELECT *, ST_Distance(geometry, ST_GeogPoint(${longitude}, ${latitude})) AS distance_m FROM \`bigquery-public-data.overture_maps.place\` - WHERE ST_DWithin(geometry, ST_GeogPoint(${longitude}, ${latitude}), ${radius}) - - `; - - if (wikidata) { - query += ` AND brand.wikidata = "${wikidata}"`; - } + let queryParts: string[] = []; - if (country) { - query += ` AND addresses.list[OFFSET(0)].element.country = "${country}"`; - } - query += ` ORDER BY distance_m LIMIT 100;`; + // Base query and distance calculation if latitude and longitude are provided + queryParts.push(`-- Overture Maps API: Get places nearby \n`); + queryParts.push(`SELECT *`); + + if (latitude && longitude) { + queryParts.push(`, ST_Distance(geometry, ST_GeogPoint(${latitude}, ${longitude})) AS distance_m`); + } + + queryParts.push(`FROM \`bigquery-public-data.overture_maps.place\``); + + // Conditional filters + let whereClauses: string[] = []; + if (latitude && longitude && radius) { + whereClauses.push(`ST_DWithin(geometry, ST_GeogPoint(${longitude}, ${latitude}), ${radius})`); + } + + if (brand_wikidata) { + whereClauses.push(`brand.wikidata = "${brand_wikidata}"`); + } + if (brand_name) { + whereClauses.push(`brand.names.primary = "${brand_name}"`); + } + + if (country) { + whereClauses.push(`addresses.list[OFFSET(0)].element.country = "${country}"`); + } + + if(categories && categories.length > 0){ + console.log(categories); + whereClauses.push(`categories.primary IN UNNEST(["${categories.join('","')}"])`); + } + + if (min_confidence) { + whereClauses.push(`confidence >= ${min_confidence}`); + } + + // Combine where clauses + if (whereClauses.length > 0) { + queryParts.push(`WHERE ${whereClauses.join(' AND ')}`); + } + + // Order by distance if latitude and longitude are provided + if (latitude && longitude) { + queryParts.push(`ORDER BY distance_m`); + } + + // Limit results if no filters are provided + if (!latitude && !longitude && !brand_wikidata && !brand_name) { + queryParts.push(`LIMIT ${this.applyMaxLimit(limit)}`); + } + + // Finalize the query stbilledAmountInnGBring + const query = queryParts.join(' ') + ';'; this.logger.debug(`Running query: ${query}`); + + const { rows } = await this.runQuery(query); + return rows.map((row: any) => this.parseRow(row)); + } - const options = { - query: query, - location: 'US', // Adjust the location if necessary - }; + applyMaxLimit(limit: number): number { + return Math.min(limit, 1000); + } - const [rows] = await this.bigQueryClient.query(options); + getDefaultLabels() : any + { - return rows.map((row: any) => (this.parseRow(row))); + const labels = { + "product": "overture-maps-api", + "env": process.env.ENV + } + return labels; } + + async runQuery(query:string, labels:any={}):Promise<{rows:any[], statistics:IQueryStatistics}> + { + + let start = Date.now() + const options = { + query: query, + // Location must match that of the dataset(s) referenced in the query. + location: 'US', + labels: {...labels, ...this.getDefaultLabels()}, + }; + // Run the query as a job + const [job] = await this.bigQueryClient.createQueryJob(options); + + // Wait for the query to finish + const [rows] = await job.getQueryResults(); + const [result] = await job.getMetadata(); + + const totalBytesProcessed = parseInt(result.statistics.totalBytesProcessed); + const totalBytesBilled = parseInt(result.statistics.query.totalBytesBilled); + + const statistics:IQueryStatistics = { + totalBytesProcessed, + totalBytesBilled, + billedAmountInGB: Math.round(totalBytesBilled / 1000000000), + bytesProcessedInGB: Math.round(totalBytesProcessed / 1000000000), + durationMs: Date.now() - start + } + + const QueryFirstLine = query.split('\n')[0]; + this.logger.log(`BigQuery: Duration: ${statistics.durationMs}ms. Billed ${statistics.billedAmountInGB} GB. Processed ${statistics.bytesProcessedInGB} GB. Query ${QueryFirstLine}`); + + return {rows,statistics}; + } } \ No newline at end of file diff --git a/src/gcs/gcs.service.ts b/src/gcs/gcs.service.ts index 988f663..43300e5 100644 --- a/src/gcs/gcs.service.ts +++ b/src/gcs/gcs.service.ts @@ -20,7 +20,7 @@ export class GcsService { this.bucket = this.storage.bucket(process.env.GCS_BUCKET_NAME); // Apply lifecycle rule to auto-delete objects after 90 days - this.setLifecyclePolicy(); + //this.setLifecyclePolicy(); } // Method to generate unique cache file names diff --git a/src/guards/is-authenticated.guard.ts b/src/guards/is-authenticated.guard.ts new file mode 100644 index 0000000..e43c610 --- /dev/null +++ b/src/guards/is-authenticated.guard.ts @@ -0,0 +1,18 @@ +import { CanActivate, ExecutionContext, HttpException, HttpStatus, Logger, UnauthorizedException } from '@nestjs/common'; +import { Request, Response } from 'express' + +export class IsAuthenticatedGuard implements CanActivate { + + canActivate(context: ExecutionContext) { + //console.log("test") + const request:Request = context.switchToHttp().getRequest(); + const user = request['user'] || request.res.locals.user; + //console.log("u",user) + + //Logger.debug("IsAuthenticatedGuard", JSON.stringify(user)) + //throw new UnauthorizedException(); + if(!user) throw new HttpException("Unauthorized - pass your API Key as the api-key header on the request", HttpStatus.UNAUTHORIZED) + return !!user; + } + +} \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index ae7011d..b733a9b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,10 +2,30 @@ import { ValidationPipe } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module'; +import * as bodyParser from 'body-parser'; async function bootstrap() { - const app = await NestFactory.create(AppModule); - app.useGlobalPipes(new ValidationPipe()); + const app = await NestFactory.create(AppModule); + app.useGlobalPipes(new ValidationPipe({forbidNonWhitelisted:true, whitelist:true})); + app.use(bodyParser.json({limit: '50mb'})); + app.use(bodyParser.urlencoded({limit: '50mb', extended: true})); + + app.use(function(req, res, next) { + res.header('x-powered-by', 'API by ThatAPICompany.com, Data by OvertureMaps.org'); + next(); + }); + + const corsOptions = { + "origin": "*", + "methods": "GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS", + "preflightContinue": false, + "optionsSuccessStatus": 204, + "credentials":true, + "allowedHeaders": 'Content-Type, Authorization, Accept, Observe, api_key', + "exposedHeaders":"Pagination-Count, Pagination-Page, Pagination-Limit, Query-Version" + } + app.enableCors(corsOptions); + app.useGlobalPipes(new ValidationPipe({transform: true})); await app.listen(8080); } bootstrap(); diff --git a/src/middleware/auth-api.middleware.ts b/src/middleware/auth-api.middleware.ts new file mode 100644 index 0000000..f920381 --- /dev/null +++ b/src/middleware/auth-api.middleware.ts @@ -0,0 +1,68 @@ +import { + Injectable, + NestMiddleware, + Logger, +} from '@nestjs/common'; + +import { Request, Response } from 'express'; +//import CacheService from '../cache/CacheService'; +import TheAuthAPI from 'theauthapi'; + +const DEMO_API_KEY = 'demo-api-key'; + +@Injectable() +export class AuthAPIMiddleware implements NestMiddleware { + private theAuthAPI: TheAuthAPI; + + constructor() { + if(process.env.AUTH_API_ACCESS_KEY)this.theAuthAPI = new TheAuthAPI(process.env.AUTH_API_ACCESS_KEY); + } + + async use(req: Request, res: Response, next: () => void) { + + + if (!req.get('X-Api-Key') && !req.get('api_key') && !req.get('api-key') || req.res.locals['user']?.id ) { + next(); + } else { + const key: string = req.get('X-Api-Key') || req.get('api_key') || req.get('api-key'); + + + if (key.toLowerCase() === DEMO_API_KEY) { + req['user'] = req.res.locals['user'] = { + metadata: { + isDemoAccount:true + }, + accountId: 'demo-account-id', + userId: 'demo-user-id', + }; + next(); + return; + } + + try { + if(!this.theAuthAPI) { + Logger.error('APIKeyMiddleware Error: Auth API not initialized'); + next(); + return; + } + const apiKey = await this.theAuthAPI.apiKeys.authenticateKey(key); + if (apiKey) { + const userObj = { + metadata: apiKey.customMetaData, + accountId: apiKey.customAccountId, + userId: apiKey.customUserId, + }; + + //set to both req and locals for backwards compatibility + req['user'] = req.res.locals['user'] = userObj; + } + next(); + return; + } catch (error) { + Logger.error('APIKeyMiddleware Error:', error, ` key: ${key}`); + } + next() + return; + } + } +} diff --git a/src/places/dto/get-brands.dto.ts b/src/places/dto/get-brands.dto.ts index c699ee9..909d880 100644 --- a/src/places/dto/get-brands.dto.ts +++ b/src/places/dto/get-brands.dto.ts @@ -1,22 +1,31 @@ // src/places/dto/get-brands.dto.ts -import { IsNumber, IsOptional, IsString, Min, ValidateIf } from 'class-validator'; +import { Transform } from 'class-transformer'; +import { IsNumber, IsOptional, IsString, Max, MaxLength, Min, MinLength, ValidateIf } from 'class-validator'; export class GetBrandsDto { @IsOptional() @IsString() - country_code?: string; // ISO 3166 country code + @MaxLength(2) + @MinLength(2) + country?: string; // ISO 3166 country code - @ValidateIf(o => !o.country_code) + @ValidateIf(o => !o.country) @IsNumber() lat?: number; - @ValidateIf(o => !o.country_code) + @ValidateIf(o => !o.country) @IsNumber() lng?: number; - @ValidateIf(o => !o.country_code) + @ValidateIf(o => !o.country) @IsOptional() @IsNumber() @Min(1) radius?: number = 1000; // Default radius is 1000 meters if not provided + + //transform into an array of strings + @IsOptional() + @Transform(({ value }) => value.split(',')) + @IsString({ each: true }) + categories?: string[]; // Array of category names } \ No newline at end of file diff --git a/src/places/dto/get-categories.dto.ts b/src/places/dto/get-categories.dto.ts new file mode 100644 index 0000000..e6d8bba --- /dev/null +++ b/src/places/dto/get-categories.dto.ts @@ -0,0 +1,26 @@ +// src/places/dto/get-brands.dto.ts +import { Transform } from 'class-transformer'; +import { IsNumber, IsOptional, IsString, Max, MaxLength, Min, MinLength, ValidateIf } from 'class-validator'; + +export class GetCategoriesDto { + @IsOptional() + @IsString() + @MaxLength(2) + @MinLength(2) + country?: string; // ISO 3166 country code + + @ValidateIf(o => !o.country) + @IsNumber() + lat?: number; + + @ValidateIf(o => !o.country) + @IsNumber() + lng?: number; + + @ValidateIf(o => !o.country) + @IsOptional() + @IsNumber() + @Min(1) + radius?: number = 1000; // Default radius is 1000 meters if not provided + +} \ No newline at end of file diff --git a/src/places/dto/get-places.dto.ts b/src/places/dto/get-places.dto.ts index dd911dd..80e23d8 100644 --- a/src/places/dto/get-places.dto.ts +++ b/src/places/dto/get-places.dto.ts @@ -1,28 +1,53 @@ // src/places/dto/get-places.dto.ts import { Transform } from 'class-transformer'; -import { IsNumber, IsOptional, IsString, Min } from 'class-validator'; +import { IsNumber, IsOptional, IsString, Min, ValidateIf } from 'class-validator'; export class GetPlacesDto { //convert string to number + @ValidateIf(o => !o.country) @Transform(({ value }) => parseFloat(value)) @IsNumber() lat: number; + @ValidateIf(o => !o.country) @Transform(({ value }) => parseFloat(value)) @IsNumber() lng: number; + @ValidateIf(o => !o.country) @IsOptional() @Transform(({ value }) => parseFloat(value)) @IsNumber() @Min(1) radius?: number = 1000; // Default radius is 1000 meters if not provided + + @IsOptional() + @Transform(({ value }) => parseInt(value)) + @IsNumber() + @Min(1) + limit?: number = 100; // Default limit is 10 if not provided + @IsOptional() @IsString() - wikidata?: string; // Wikidata brand ID + country?: string; // ISO 3166 country code @IsOptional() @IsString() - country?: string; // ISO 3166 country code + brand_wikidata?: string; // Wikidata brand ID + + @IsOptional() + @IsString() + brand_name?: string; // Wikidata brand ID + + @IsOptional() + @Transform(({ value }) => parseFloat(value)) + min_confidence?: number = 0.5; + + + //transform into an array of strings + @IsOptional() + @Transform(({ value }) => String(value).split(',')) + @IsString({ each: true }) + categories?: string[]; // Array of category names } diff --git a/src/places/places.controller.ts b/src/places/places.controller.ts index 5e4cec1..f4bc03a 100644 --- a/src/places/places.controller.ts +++ b/src/places/places.controller.ts @@ -1,15 +1,18 @@ // src/places/places.controller.ts -import { Controller, Get, Logger, Query } from '@nestjs/common'; +import { Controller, Get, Logger, Query, UseGuards } from '@nestjs/common'; import { BigQueryService } from '../bigquery/bigquery.service'; import { GcsService } from '../gcs/gcs.service'; import { GetPlacesDto } from './dto/get-places.dto'; import { PlaceResponseDto } from './dto/place-response.dto'; import { GetBrandsDto } from './dto/get-brands.dto'; +import { IsAuthenticatedGuard } from '../guards/is-authenticated.guard'; @Controller('places') +@UseGuards(IsAuthenticatedGuard) export class PlacesController { - logger = new Logger('PlacesController'); + logger = new Logger('PlacesController'); + constructor( private readonly bigQueryService: BigQueryService, private readonly gcsService: GcsService, @@ -17,7 +20,8 @@ export class PlacesController { @Get() async getPlaces(@Query() query: GetPlacesDto) { - const { lat, lng, radius, wikidata, country } = query; + + const { lat, lng, radius, country, min_confidence, brand_wikidata,brand_name,categories,limit } = query; const cacheKey = `get-places-${JSON.stringify(query)}`; @@ -27,17 +31,20 @@ export class PlacesController { return cachedResult.map((place: any) => new PlaceResponseDto(place)); } + // if only country is provided, then potentially just use the lat / lng of it's capital city + // If no cache, query BigQuery with wikidata and country support - const places = await this.bigQueryService.getPlacesNearby(lat, lng, radius, wikidata, country); + const places = await this.bigQueryService.getPlacesNearby(lat, lng, radius, brand_wikidata,brand_name, country, categories, min_confidence,limit); // Cache the results in GCS - await this.gcsService.storeJSON (places,cacheKey); + //await this.gcsService.storeJSON (places,cacheKey); return places.map((place: any) => new PlaceResponseDto(place)); } - @Get('brands') + + @Get('brands') async getBrands(@Query() query: GetBrandsDto) { - const { country_code, lat, lng, radius } = query; + const { country, lat, lng, radius, categories } = query; const cacheKey = `get-places-brands-${JSON.stringify(query)}`; @@ -47,7 +54,7 @@ export class PlacesController { return cachedResult; } - const brands = await this.bigQueryService.getBrandsNearby(country_code, lat, lng, radius); + const brands = await this.bigQueryService.getBrandsNearby(country, lat, lng, radius, categories); return brands; } diff --git a/test/app.e2e-spec.ts b/test/app.e2e-spec.ts index 50cda62..ff45807 100644 --- a/test/app.e2e-spec.ts +++ b/test/app.e2e-spec.ts @@ -19,6 +19,6 @@ describe('AppController (e2e)', () => { return request(app.getHttpServer()) .get('/') .expect(200) - .expect('Hello World!'); + .expect('API by ThatAPICompany.com, Data by OvertureMaps.org'); }); });