Merge pull request #10 from jack-richards/development

Added alwaysQuerySnapshotAPI setting.

README.md

@@ -73,6 +73,7 @@ To configure the application you need to specify the values for all the fields i
         "buy": 5,
         "sell": -8
     },
+    "alwaysQuerySnapshotAPI": true,
     "excludedSteamIDs": [
         "76561199384015307"
     ],
@@ -127,6 +128,32 @@ A list of descriptions that, when detected within a listing's details, causes th
 ]
 ```
 
+### `alwaysQuerySnapshotAPI`
+
+This setting determines whether the pricer should consistently call the snapshot API for each item during the pricing process, regardless of the number of listings available in the database. By default, this setting is set to `true`.
+
+#### Behavior:
+
+- **true**: The pricer always calls the snapshot API for every item, ensuring the most up-to-date information. However, this may result in slower pricing processes due to the API rate limits.
+
+- **false**: The pricer avoids unnecessary API calls if there are already a suitable number of listings in the database to generate a price. This speeds up the pricing process significantly as it bypasses the need to call the snapshot API. However, there are trade-offs to consider:
+
+#### Trade-offs:
+
+1. **Accuracy vs. Speed**: By setting `alwaysQuerySnapshotAPI` to `false`, the pricing process becomes faster but may use a smaller pool of listings to calculate prices. The logic ensures that if there are at least 10 buy listings and 1 sell listing, no further data retrieval is attempted.
+
+2. **Timeliness of Data**: When the API is not called every time, there's a risk of using listings with prices that are up to approximately 35 minutes old, in the worst-case scenario.
+
+#### Benefits:
+
+- **Improved Performance**: Setting `alwaysQuerySnapshotAPI` to `false` makes the pricer more efficient, especially when dealing with large item lists, as it avoids the minimum 2-second wait time per item.
+
+Consider your requirements for pricing accuracy, speed, and data freshness when configuring this setting.
+
+```JSON
+"alwaysQuerySnapshotAPI": true
+```
+
 ## API Routes & Socket IO
 The socket io server will emit events called 'price' with an item object as the value. The item objects are structured like the following:
 ```JSON
@@ -240,4 +267,3 @@ pm2 start bptf-autopricer.js
 - **Manual Additions:** Add items to price manually to [`item_list.json`](https://github.com/jack-richards/bptf-autopricer/blob/main/files/item_list.json). Make sure to use the valid format shown.
 
 Each item name given should be the same as the one used for its listings on [backpack.tf](https://backpack.tf/). For example, `Non-Craftable Tour of Duty Ticket` **NOT** `Uncraftable Tour of Duty Ticket`.
-

bptf-autopricer.js

@@ -33,6 +33,8 @@ const excludedListingDescriptions = config.excludedListingDescriptions;
 // Blocked attributes that we want to ignore. (Paints, parts, etc.)
 const blockedAttributes = config.blockedAttributes;
 
+const alwaysQuerySnapshotAPI = config.alwaysQuerySnapshotAPI;
+
 // Create database instance for pg-promise.
 const pgp = require('pg-promise')({
     schema: config.database.schema
@@ -138,6 +140,44 @@ rws.addEventListener('open', event => {
     console.log('Connected to socket.');
 });
 
+const countListingsForItem = async (name) => {
+    try {
+        const result = await db.one(`
+            SELECT 
+                COUNT(*) FILTER (WHERE intent = 'sell') AS sell_count,
+                COUNT(*) FILTER (WHERE intent = 'buy') AS buy_count
+            FROM listings
+            WHERE name = $1;
+        `, [name]);
+
+        const { sell_count, buy_count } = result;
+        return sell_count >= 1 && buy_count >= 10;
+    } catch (error) {
+        console.error("Error counting listings:", error);
+        throw error;
+    }
+};
+
+const updateFromSnapshot = async (name, sku) => {
+    // Check if always call snapshot API setting is enabled.
+    if (!alwaysQuerySnapshotAPI) {
+        // Check if required number of listings already exist in database for item.
+        // If not we need to query the snapshots API.
+        let callSnapshot = await countListingsForItem(name);
+        if(callSnapshot) {
+            // Get listings from snapshot API.
+            let unformatedListings = await Methods.getListingsFromSnapshots(name);
+            // Insert snapshot listings.
+            await insertListings(unformatedListings, sku, name);
+        }
+    } else {
+        // Get listings from snapshot API.
+        let unformatedListings = await Methods.getListingsFromSnapshots(name);
+        // Insert snapshot listings.
+        await insertListings(unformatedListings, sku, name);
+    }
+}
+
 const calculateAndEmitPrices = async () => {
     let item_objects = [];
     for (const name of allowedItemNames) {
@@ -148,12 +188,11 @@ const calculateAndEmitPrices = async () => {
             }
             // Get sku of item via the item name.
             let sku = schemaManager.schema.getSkuFromName(name);
-            // Get listings from snapshot API.
-            let unformatedListings = await Methods.getListingsFromSnapshots(name);
-            // Insert snapshot listings.
-            await insertListings(unformatedListings, sku, name);
-            // Now we have the latest snapshot data we start to determine
-            // the price of the item.
+            // Delete old listings from database.
+            await deleteOldListings();
+            // Use snapshot API to populate database with listings for item.
+            await updateFromSnapshot(name, sku);
+            // Start process of pricing item.
             let arr = await determinePrice(name, sku);
             let item = finalisePrice(arr, name, sku);
             // If the item is undefined, we skip it.
@@ -731,4 +770,4 @@ const finalisePrice = (arr, name, sku) => {
     }
 };
 
-listen();
+listen();

config.json

@@ -15,6 +15,7 @@
         "buy": 5,
         "sell": -8
     },
+    "alwaysQuerySnapshotAPI": true,
     "excludedSteamIDs": [
         "76561199384015307",
         "76561199495073910",