diff --git a/stats-backend/api2/migrations/0015_offer_hourly_price_glm_offer_hourly_price_usd.py b/stats-backend/api2/migrations/0015_offer_hourly_price_glm_offer_hourly_price_usd.py new file mode 100644 index 0000000..34f0b27 --- /dev/null +++ b/stats-backend/api2/migrations/0015_offer_hourly_price_glm_offer_hourly_price_usd.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.7 on 2024-02-23 22:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0014_node_uptime_created_at'), + ] + + operations = [ + migrations.AddField( + model_name='offer', + name='hourly_price_glm', + field=models.FloatField(blank=True, null=True), + ), + migrations.AddField( + model_name='offer', + name='hourly_price_usd', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/stats-backend/api2/models.py b/stats-backend/api2/models.py index 2d6912f..e641c8f 100644 --- a/stats-backend/api2/models.py +++ b/stats-backend/api2/models.py @@ -36,6 +36,8 @@ class Offer(models.Model): created_at = models.DateTimeField(auto_now_add=True) monthly_price_glm = models.FloatField(null=True, blank=True) monthly_price_usd = models.FloatField(null=True, blank=True) + hourly_price_glm = models.FloatField(null=True, blank=True) + hourly_price_usd = models.FloatField(null=True, blank=True) is_overpriced = models.BooleanField(default=False) overpriced_compared_to = models.ForeignKey( EC2Instance, on_delete=models.CASCADE, null=True diff --git a/stats-backend/api2/scanner.py b/stats-backend/api2/scanner.py index e33f4a9..048ae7b 100644 --- a/stats-backend/api2/scanner.py +++ b/stats-backend/api2/scanner.py @@ -28,171 +28,89 @@ def update_providers_info(node_props): now = timezone.now() days_in_current_month = calendar.monthrange(now.year, now.month)[1] seconds_current_month = days_in_current_month * 24 * 60 * 60 + hours_in_current_month = days_in_current_month * 24 glm_usd_value = GLM.objects.get(id=1) - print(f"Updating {len(node_props)} providers") for prop in node_props: data = json.loads(prop) provider_id = data["node_id"] wallet = data["wallet"] unique_providers.add(provider_id) # Add provider to the set obj, created = Node.objects.get_or_create(node_id=provider_id) - if created: - print(f"Created new provider: {prop}") - offerobj = Offer.objects.create( - properties=data, provider=obj, runtime=data["golem.runtime.name"] - ) - if data["golem.runtime.name"] == "vm": - vectors = {} - for key, value in enumerate(data["golem.com.usage.vector"]): - vectors[value] = key - monthly_pricing = ( - ( - data["golem.com.pricing.model.linear.coeffs"][ - vectors["golem.usage.duration_sec"] - ] - * seconds_current_month - ) - + ( - data["golem.com.pricing.model.linear.coeffs"][ - vectors["golem.usage.cpu_sec"] - ] - * seconds_current_month - * data["golem.inf.cpu.threads"] - ) - + data["golem.com.pricing.model.linear.coeffs"][-1] + offerobj, offercreated = Offer.objects.get_or_create( + provider=obj, runtime=data["golem.runtime.name"] + ) + if data["golem.runtime.name"] == "vm": + vectors = {} + for key, value in enumerate(data["golem.com.usage.vector"]): + vectors[value] = key + monthly_pricing = ( + ( + data["golem.com.pricing.model.linear.coeffs"][ + vectors["golem.usage.duration_sec"] + ] + * seconds_current_month ) - if not monthly_pricing: - print(f"Monthly price is {monthly_pricing}") - offerobj.monthly_price_glm = monthly_pricing - offerobj.monthly_price_usd = ( - monthly_pricing * glm_usd_value.current_price + + ( + data["golem.com.pricing.model.linear.coeffs"][ + vectors["golem.usage.cpu_sec"] + ] + * seconds_current_month + * data["golem.inf.cpu.threads"] ) - vcpu_needed = data.get("golem.inf.cpu.threads", 0) - memory_needed = data.get("golem.inf.mem.gib", 0.0) - closest_ec2 = ( - EC2Instance.objects.annotate( - cpu_diff=Abs(F("vcpu") - vcpu_needed), - memory_diff=Abs(F("memory") - memory_needed), - ) - .order_by("cpu_diff", "memory_diff", "price_usd") - .first() + + data["golem.com.pricing.model.linear.coeffs"][-1] + ) + if not monthly_pricing: + print(f"Monthly price is {monthly_pricing}") + offerobj.monthly_price_glm = monthly_pricing + offerobj.monthly_price_usd = monthly_pricing * glm_usd_value.current_price + offerobj.hourly_price_glm = monthly_pricing / hours_in_current_month + offerobj.hourly_price_usd = ( + offerobj.monthly_price_usd / hours_in_current_month + ) + vcpu_needed = data.get("golem.inf.cpu.threads", 0) + memory_needed = data.get("golem.inf.mem.gib", 0.0) + closest_ec2 = ( + EC2Instance.objects.annotate( + cpu_diff=Abs(F("vcpu") - vcpu_needed), + memory_diff=Abs(F("memory") - memory_needed), ) + .order_by("cpu_diff", "memory_diff", "price_usd") + .first() + ) - # Compare and update the Offer object - if closest_ec2 and monthly_pricing: - offer_price_usd = monthly_pricing * glm_usd_value.current_price - ec2_monthly_price = closest_ec2.price_usd * 730 - - offer_is_more_expensive = offer_price_usd > ec2_monthly_price - offer_is_cheaper = offer_price_usd < ec2_monthly_price - - # Update Offer object fields for expensive comparison - offerobj.is_overpriced = offer_is_more_expensive - offerobj.overpriced_compared_to = ( - closest_ec2 if offer_is_more_expensive else None - ) - offerobj.times_more_expensive = ( - offer_price_usd / float(ec2_monthly_price) - if offer_is_more_expensive - else None - ) - - # Update Offer object fields for cheaper comparison - offerobj.cheaper_than = closest_ec2 if offer_is_cheaper else None - offerobj.times_cheaper = ( - float(ec2_monthly_price) / offer_price_usd - if offer_is_cheaper - else None - ) - - else: - # print( - # "No matching EC2Instance found or monthly pricing is not available." - # ) - offerobj.is_overpriced = False - offerobj.overpriced_compared_to = None - offerobj.save() - obj.wallet = wallet - # Verify each node's status - is_online = check_node_status(obj.node_id) + # Compare and update the Offer object + if closest_ec2 and monthly_pricing: + offer_price_usd = monthly_pricing * glm_usd_value.current_price + ec2_monthly_price = closest_ec2.price_usd * 730 - obj.online = is_online - obj.save() - else: - offerobj, offercreated = Offer.objects.get_or_create( - provider=obj, runtime=data["golem.runtime.name"] - ) - if data["golem.runtime.name"] == "vm": - vectors = {} - for key, value in enumerate(data["golem.com.usage.vector"]): - vectors[value] = key - monthly_pricing = ( - ( - data["golem.com.pricing.model.linear.coeffs"][ - vectors["golem.usage.duration_sec"] - ] - * seconds_current_month - ) - + ( - data["golem.com.pricing.model.linear.coeffs"][ - vectors["golem.usage.cpu_sec"] - ] - * seconds_current_month - * data["golem.inf.cpu.threads"] - ) - + data["golem.com.pricing.model.linear.coeffs"][-1] + offer_is_more_expensive = offer_price_usd > ec2_monthly_price + offer_is_cheaper = offer_price_usd < ec2_monthly_price + + # Update Offer object fields for expensive comparison + offerobj.is_overpriced = offer_is_more_expensive + offerobj.overpriced_compared_to = ( + closest_ec2 if offer_is_more_expensive else None ) - if not monthly_pricing: - print(f"Monthly price is {monthly_pricing}") - offerobj.monthly_price_glm = monthly_pricing - offerobj.monthly_price_usd = ( - monthly_pricing * glm_usd_value.current_price + offerobj.times_more_expensive = ( + offer_price_usd / float(ec2_monthly_price) + if offer_is_more_expensive + else None ) - vcpu_needed = data.get("golem.inf.cpu.threads", 0) - memory_needed = data.get("golem.inf.mem.gib", 0.0) - closest_ec2 = ( - EC2Instance.objects.annotate( - cpu_diff=Abs(F("vcpu") - vcpu_needed), - memory_diff=Abs(F("memory") - memory_needed), - ) - .order_by("cpu_diff", "memory_diff", "price_usd") - .first() + # Update Offer object fields for cheaper comparison + offerobj.cheaper_than = closest_ec2 if offer_is_cheaper else None + offerobj.times_cheaper = ( + float(ec2_monthly_price) / offer_price_usd + if offer_is_cheaper + else None ) - # Compare and update the Offer object - if closest_ec2 and monthly_pricing: - offer_price_usd = monthly_pricing * glm_usd_value.current_price - ec2_monthly_price = closest_ec2.price_usd * 730 - - offer_is_more_expensive = offer_price_usd > ec2_monthly_price - offer_is_cheaper = offer_price_usd < ec2_monthly_price - - # Update Offer object fields for expensive comparison - offerobj.is_overpriced = offer_is_more_expensive - offerobj.overpriced_compared_to = ( - closest_ec2 if offer_is_more_expensive else None - ) - offerobj.times_more_expensive = ( - offer_price_usd / float(ec2_monthly_price) - if offer_is_more_expensive - else None - ) - - # Update Offer object fields for cheaper comparison - offerobj.cheaper_than = closest_ec2 if offer_is_cheaper else None - offerobj.times_cheaper = ( - float(ec2_monthly_price) / offer_price_usd - if offer_is_cheaper - else None - ) - - else: - # print( - # "No matching EC2Instance found or monthly pricing is not available." - # ) - offerobj.is_overpriced = False - offerobj.overpriced_compared_to = None + else: + # print( + # "No matching EC2Instance found or monthly pricing is not available." + # ) + offerobj.is_overpriced = False + offerobj.overpriced_compared_to = None offerobj.properties = data offerobj.save() diff --git a/stats-backend/api2/serializers.py b/stats-backend/api2/serializers.py index 8ab1755..5cd62ee 100644 --- a/stats-backend/api2/serializers.py +++ b/stats-backend/api2/serializers.py @@ -23,6 +23,8 @@ class Meta: "properties", "updated_at", "monthly_price_usd", + "hourly_price_usd", + "hourly_price_glm", "is_overpriced", "overpriced_compared_to", "suggest_env_per_hour_price", diff --git a/stats-backend/api2/tasks.py b/stats-backend/api2/tasks.py index 7e9436e..d5eb6a7 100644 --- a/stats-backend/api2/tasks.py +++ b/stats-backend/api2/tasks.py @@ -30,11 +30,38 @@ @app.task def v2_network_online_to_redis(): - data = Node.objects.filter(online=True) - serializer = NodeSerializer(data, many=True) - test = json.dumps(serializer.data, default=str) + # Fetch and process data from the external domain + response = requests.get( + "https://reputation.dev-test.golem.network/v1/providers/scores" + ) + if response.status_code == 200: + external_data = response.json() + success_rate_mapping = { + provider["providerId"]: provider["scores"]["successRate"] + for provider in external_data["providers"] + } + + # Fetch your existing nodes + data = Node.objects.filter(online=True) + serializer = NodeSerializer(data, many=True) + serialized_data = serializer.data - r.set("v2_online", test) + # Attach successRate to each node if the providerId matches + for node in serialized_data: + node_id = node["node_id"] + if node_id in success_rate_mapping: + node["taskReputation"] = success_rate_mapping[node_id] + else: + node["taskReputation"] = None + + # Serialize and save to Redis + test = json.dumps(serialized_data, default=str) + r.set("v2_online", test) + else: + print( + "Failed to retrieve data from the reputation system!", response.status_code + ) + pass @app.task