update roadmap for Building shapes

README.md

@@ -25,7 +25,7 @@
 - [x] Places/Brands endpoint
 - [x] Places/Categories endpoint
 - [x] Places/Countries endpoint
-- [ ] Places/Buildings endpoint
+- [ ] Places/Buildings endpoint - very expensive dataset in BigQuery so will require sharding by country and optimizing
 - [ ] Addresses endpoint for Overture Maps 'Address' Theme
 - [ ] Base endpoint for Overture Maps 'Base' Theme
 - [ ] Buildings endpoint for Overture Maps 'Building' Theme

docs/auth-api.md

@@ -0,0 +1,3 @@
+# Setting Up The Auth API
+
+This will enable you to give each of your users an API key, which they can use to authenticate with your API, and control their rate-limiting to prevent cost overrun.

docs/google-cloud-platform.md

@@ -37,6 +37,12 @@ FROM `bigquery-public-data.overture_maps.place`
 WHERE ST_DWithin(geometry, ST_GeogPoint(16.3738, 48.2082), 500)
 ```
 
+### Recommended Strategy to reduce costs
+
+As there is one large table for each theme of data, a simple strategy to reduce costs is to create a new dataset in BigQuery, and copy the tables you need filtering down to just the country you are interested in. Another option would be the shard the tables by country, and then only query the tables you need.
+
+Using a metadata store such as Firebase in Datastore mode - this would allow storing of building geomtries and other data matched to place that is too expensive to query in BigQuery.
+
 ### Service Account roles
 
 For a service account in GCP that a Cloud Run instance will use to access BigQuery and Google Cloud Storage (GCS), you’ll need to grant it specific roles to ensure it has permissions to create and run BigQuery jobs, as well as read and write files in a GCS bucket. Here are the recommended roles:

src/bigquery/bigquery.service.ts

@@ -9,10 +9,14 @@ interface IQueryStatistics {
   billedAmountInGB: number;
   bytesProcessedInGB: number;
   durationMs: number;
+  costInUSD: number;
 }
 
 const MAX_LIMIT = 100000;
 
+// replace this with your own dataset if you have optimised ot to only include the country you are interested in
+const SOURCE_DATASET="bigquery-public-data.overture_maps" 
+
 @Injectable()
 export class BigQueryService {
   private bigQueryClient: BigQuery;
@@ -110,7 +114,7 @@ export class BigQueryService {
 
     let query = `-- Overture Maps API: Get brands nearby
       SELECT DISTINCT brand , count(id) as count_places
-      FROM \`bigquery-public-data.overture_maps.place\`
+      FROM \`${SOURCE_DATASET}.place\`
     `;
 
     if (country_code) {
@@ -148,7 +152,7 @@ export class BigQueryService {
   async getPlaceCountsByCountry(): Promise<{ country: string; counts:{ places:number, brands:number} }[]> {
     const query = `-- Overture Maps API: Get place counts by country
       SELECT addresses.list[OFFSET(0)].element.country AS country, COUNT(id) AS count_places, count(DISTINCT brand.names.primary ) as count_brands
-      FROM \`bigquery-public-data.overture_maps.place\`
+      FROM \`${SOURCE_DATASET}.place\`
       GROUP BY country
       ORDER BY count_places DESC;
     `;
@@ -169,7 +173,7 @@ export class BigQueryService {
       SELECT DISTINCT categories.primary AS category_primary,
       count(1) as count_places,
       count(distinct brand.names.primary) as count_brands
-      FROM \`bigquery-public-data.overture_maps.place\`
+      FROM \`${SOURCE_DATASET}.place\`
       WHERE categories.primary IS NOT NULL
     `;
     if (country) {
@@ -191,6 +195,63 @@ export class BigQueryService {
     }));
   }
 
+  async getBuildingsNearby(
+    latitude: number,
+    longitude: number,
+    radius: number = 1000,
+    min_confidence?: number,
+    limit?: number
+  ): Promise<Place[]> {
+
+    let queryParts: string[] = [];
+
+    // Base query and distance calculation if latitude and longitude are provided
+    queryParts.push(`-- Overture Maps API: Get buildings nearby \n`);
+    queryParts.push(`SELECT *`);
+
+    if (latitude && longitude) {
+      queryParts.push(`, ST_Distance(geometry, ST_GeogPoint(${latitude}, ${longitude})) AS distance_m`);
+    }
+
+    queryParts.push(`FROM \`${SOURCE_DATASET}.place\``);
+
+    // Conditional filters
+    let whereClauses: string[] = [];
+
+    if (latitude && longitude && radius) {
+      whereClauses.push(`ST_DWithin(geometry, ST_GeogPoint(${longitude}, ${latitude}), ${radius})`);
+    } 
+
+    whereClauses.push(`categories.primary = "building"`);
+
+    if (min_confidence) {
+      whereClauses.push(`confidence >= ${min_confidence}`);
+    }
+
+    // Combine where clauses
+    if (whereClauses.length > 0) {
+      queryParts.push(`WHERE ${whereClauses.join(' AND ')}`);
+    }
+
+    // Order by distance if latitude and longitude are provided
+    if (latitude && longitude) {
+      queryParts.push(`ORDER BY distance_m`);
+    }
+
+    // Limit results if no filters are provided
+    if (limit) {
+      queryParts.push(`LIMIT ${this.applyMaxLimit(limit)}`);
+    }
+
+    // Finalize the query
+    const query = queryParts.join(' ') + ';';
+    this.logger.debug(`Running query: ${query}`);
+
+    const { rows } = await this.runQuery(query);
+    return rows.map((row: any) => this.parsePlaceRow(row));
+  }
+
+
   async getPlacesNearby(
     latitude: number,
     longitude: number,
@@ -213,7 +274,7 @@ export class BigQueryService {
       queryParts.push(`, ST_Distance(geometry, ST_GeogPoint(${latitude}, ${longitude})) AS distance_m`);
     }
 
-    queryParts.push(`FROM \`bigquery-public-data.overture_maps.place\``);
+    queryParts.push(`FROM \`${SOURCE_DATASET}.place\``);
 
     // Conditional filters
     let whereClauses: string[] = [];
@@ -304,11 +365,12 @@ export class BigQueryService {
           totalBytesBilled,
           billedAmountInGB: Math.round(totalBytesBilled / 1000000000),
           bytesProcessedInGB: Math.round(totalBytesProcessed / 1000000000),
-          durationMs: Date.now() - start
+          durationMs: Date.now() - start,
+          costInUSD: Math.round(totalBytesBilled / 1000000000) * 5
       }
 
       const QueryFirstLine = query.split('\n')[0];
-      this.logger.log(`BigQuery: Duration: ${statistics.durationMs}ms. Billed ${statistics.billedAmountInGB} GB. Processed ${statistics.bytesProcessedInGB} GB. Query ${QueryFirstLine}`);
+      this.logger.log(`BigQuery: Duration: ${statistics.durationMs}ms. Billed ${statistics.billedAmountInGB} GB. USD $${statistics.costInUSD}. Query Line 1: ${QueryFirstLine}`);
 
       return {rows,statistics};
     }

src/gcs/gcs.service.ts

@@ -41,6 +41,9 @@ export class GcsService {
       const exists = await file.exists();
       if (exists[0]) {
         const [content] = await file.download();
+        //get size of file
+        const fileSize = content.length;
+        const fileSizeInKB = fileSize / 1024;
         return JSON.parse(content.toString());
       }
     } catch (error) {

src/middleware/auth-api.middleware.ts

@@ -8,7 +8,7 @@ import { Request, Response } from 'express';
 //import CacheService from '../cache/CacheService';
 import TheAuthAPI from 'theauthapi';
 
-const DEMO_API_KEY = 'demo-api-key';
+const DEMO_API_KEY = process.env.DEMO_API_KEY || 'DEMO-API-KEY';
 
 @Injectable()
 export class AuthAPIMiddleware implements NestMiddleware {
@@ -51,20 +51,10 @@ export class AuthAPIMiddleware implements NestMiddleware {
     if (!apiKeyString || req.res.locals['user']?.id ) {
       next();
     } else {
-      //if demo key, set user to demo user
-      if (apiKeyString.toLowerCase() === DEMO_API_KEY) {
-        req['user'] = req.res.locals['user'] = {
-          metadata: {
-            isDemoAccount:true
-          },
-          accountId: 'demo-account-id',
-          userId: 'demo-user-id',
-        };
-        next();
-        return;
-      }
 
       try {
+
+        // if theAuthAPI.com is integrated, check the key
         if(this.theAuthAPI) {
 
           const apiKey = await this.theAuthAPI.apiKeys.authenticateKey(apiKeyString); 
@@ -85,6 +75,19 @@ export class AuthAPIMiddleware implements NestMiddleware {
         Logger.error('APIKeyMiddleware Error:', error, ` key: ${apiKeyString}`);
       }
 
+      //if demo key, set user to demo user
+      if (apiKeyString === DEMO_API_KEY) {
+        req['user'] = req.res.locals['user'] = {
+          metadata: {
+            isDemoAccount:true
+          },
+          accountId: 'demo-account-id',
+          userId: 'demo-user-id',
+        };
+        next();
+        return;
+      }
+
       //if we got this far and they passed a key we should tell the user their key doesn't work and to check for a spelling mistake
       res.status(401).send('Unauthorized - check the spelling of your API Key');