diff --git a/markdown/2024_insights/03_analysis.md b/markdown/2024_insights/03_analysis.md index bc1a506..5c40c94 100644 --- a/markdown/2024_insights/03_analysis.md +++ b/markdown/2024_insights/03_analysis.md @@ -14,8 +14,14 @@ jupyter: ```python import json +from datetime import datetime import pandas as pd + +# Generate the current date +generated_date = datetime.now().strftime("%Y-%m-%d") + +print(f"Data generated on: {generated_date}") ``` ```python @@ -25,13 +31,14 @@ df_2023 = pd.read_csv('../../data/2024_insights/processed/nvd_data_2023.csv') df_2024.head() ``` -# Overview Metrics +## Overview Metrics -This section calculates high-level aggregate metrics to summarize vulnerability data: +This section summarizes high-level trends in vulnerability publication for 2023 and 2024 to provide a snapshot of key patterns. -1. **Total Vulnerabilities Published**: Total CVEs published in each year (2023 and 2024). -2. **Month with Most Vulnerabilities Published**: Identifies the month with the highest vulnerability count. -3. **Vulnerability Severity Distribution**: Calculates the distribution of Critical, High, Medium, and Low severity vulnerabilities. +Key insights calculated: +1. **Total Vulnerabilities Published**: How many CVEs were published each year and their percentage change. +2. **Month with Most Vulnerabilities Published**: Identifies peak publication months to understand seasonal patterns. +3. **Vulnerability Severity Distribution**: Examines the distribution of CVEs by severity (Critical, High, Medium, Low) for risk prioritization. ```python @@ -74,9 +81,21 @@ severity_distribution = { } overview_metrics = { - "total_vulnerabilities": total_vulnerabilities, - "month_with_most_vulnerabilities": month_with_most_vulnerabilities, - "severity_distribution": severity_distribution + "metadata": { + "description": "Overview metrics summarizing vulnerability data for 2023 and 2024.", + "generated_on": generated_date, + "source": ["NVD", "CISA KEV"], + "attribution": { + "NVD": "This product uses the NVD API but is not endorsed or certified by the NVD.", + "CISA KEV": "Data from CISA KEV is used under the Creative Commons 0 1.0 License." + }, + "author": "2024 Vulnerability Insights Project" + }, + "data": { + "total_vulnerabilities": total_vulnerabilities, + "month_with_most_vulnerabilities": month_with_most_vulnerabilities, + "severity_distribution": severity_distribution + } } # Save the overview_metrics to a JSON file @@ -86,13 +105,13 @@ with open("../../data/2024_insights/output/overview_metrics.json", "w") as f: overview_metrics ``` -# Time-Series Metrics +## Time-Series Metrics -This section focuses on trends over time to analyze how vulnerabilities evolve month-to-month: +Understanding vulnerability trends over time helps identify patterns in CVE publication and potential shifts in the security landscape. This section analyzes: -1. **Monthly Vulnerability Counts**: Monthly counts of vulnerabilities published in 2023 and 2024. -2. **Severity Trends**: Tracks changes in vulnerability severities (Critical, High, Medium, Low) over time. -3. **Spike in Vulnerability Counts**: Highlights months with significant increases in vulnerability publication. +1. **Monthly Vulnerability Counts**: Compare month-to-month vulnerability counts in 2023 and 2024. +2. **Severity Trends**: Examine how vulnerability severities evolved over time. +3. **Spike in Vulnerability Counts**: Highlight months with sharp increases to correlate with external events or publication cycles. ```python @@ -146,9 +165,21 @@ spike_analysis = { } } time_series_metrics = { - "monthly_vulnerability_counts": monthly_vulnerability_counts, - "severity_trends": severity_trends, - "spike_analysis": spike_analysis, + "metadata": { + "description": "Time-series analysis of monthly vulnerability trends for 2023 and 2024.", + "generated_on": generated_date, + "source": ["NVD", "CISA KEV"], + "attribution": { + "NVD": "This product uses the NVD API but is not endorsed or certified by the NVD.", + "CISA KEV": "Data from CISA KEV is used under the Creative Commons 0 1.0 License." + }, + "author": "2024 Vulnerability Insights Project" + }, + "data": { + "monthly_vulnerability_counts": monthly_vulnerability_counts, + "severity_trends": severity_trends, + "spike_analysis": spike_analysis, + } } # Save the time_series_metrics to a JSON file @@ -158,14 +189,14 @@ with open("../../data/2024_insights/output/time_series_metrics.json", "w") as f: time_series_metrics ``` -# Vendor/Product Analysis +## Vendor/Product Analysis -This section investigates vulnerabilities at the vendor and product levels: +Analyzing vulnerabilities by vendor and product provides actionable insights for vendor risk management and prioritizing patches. Key metrics include: -1. **Top Vendors by Vulnerabilities**: Ranks vendors with the most vulnerabilities, focusing on Critical and High severities. -2. **Top Products by Vulnerabilities**: Visualizes vulnerabilities grouped by product, nested under their vendors. -3. **Vulnerability Trends by Vendor**: Tracks monthly trends for top vendors across 2023 and 2024. -4. **Critical Vulnerability Spike Analysis**: Explores Critical vulnerabilities by vendor and product for peak months. +1. **Top Vendors by Vulnerabilities**: Identify vendors with the most vulnerabilities, focusing on high-severity issues. +2. **Top Products by Vulnerabilities**: Drill down into specific products with the highest vulnerability counts. +3. **Vulnerability Trends by Vendor**: Explore how vulnerabilities evolve for major vendors over time. +4. **Critical Vulnerability Spike Analysis**: Detect peak months for critical vulnerabilities by vendor and product to focus remediation efforts. ```python @@ -241,75 +272,78 @@ critical_vulns_2024 = df_2024[df_2024['CVSS_Severity'] == 'CRITICAL'] for vendor in top_vendors_2024['Vendor']: critical_spikes["2024"][vendor] = {} - vendor_critical = critical_vulns_2024[critical_vulns_2024['Vendor'] == vendor] - for product in top_products["2024"].get(vendor, []): - product_name = product['Product'] - product_critical = vendor_critical[vendor_critical['Product'] == product_name] - if not product_critical.empty: - top_month = ( - product_critical.groupby('Published_Month').size().idxmax() - ) - count = ( - product_critical.groupby('Published_Month').size().max() - ) - critical_spikes["2024"][vendor][product_name] = { - "month": pd.to_datetime(f"2024-{top_month}-01").strftime('%B'), - "count": int(count) - } +vendor_critical = critical_vulns_2024[critical_vulns_2024['Vendor'] == vendor] +for product in top_products["2024"].get(vendor, []): + product_name = product['Product'] +product_critical = vendor_critical[vendor_critical['Product'] == product_name] +if not product_critical.empty: + top_month = ( + product_critical.groupby('Published_Month').size().idxmax() + ) +count = ( + product_critical.groupby('Published_Month').size().max() +) +critical_spikes["2024"][vendor][product_name] = { + "month": pd.to_datetime(f"2024-{top_month}-01").strftime('%B'), + "count": int(count) +} for vendor in top_vendors_2023['Vendor']: critical_spikes["2023"][vendor] = {} - vendor_critical = critical_vulns_2023[critical_vulns_2023['Vendor'] == vendor] - for product in top_products["2023"].get(vendor, []): - product_name = product['Product'] - product_critical = vendor_critical[vendor_critical['Product'] == product_name] - if not product_critical.empty: - top_month = ( - product_critical.groupby('Published_Month').size().idxmax() - ) - count = ( - product_critical.groupby('Published_Month').size().max() - ) - critical_spikes["2023"][vendor][product_name] = { - "month": pd.to_datetime(f"2023-{top_month}-01").strftime('%B'), - "count": int(count) - } +vendor_critical = critical_vulns_2023[critical_vulns_2023['Vendor'] == vendor] +for product in top_products["2023"].get(vendor, []): + product_name = product['Product'] +product_critical = vendor_critical[vendor_critical['Product'] == product_name] +if not product_critical.empty: + top_month = ( + product_critical.groupby('Published_Month').size().idxmax() + ) +count = ( + product_critical.groupby('Published_Month').size().max() +) +critical_spikes["2023"][vendor][product_name] = { + "month": pd.to_datetime(f"2023-{top_month}-01").strftime('%B'), + "count": int(count) +} # Final Vendor/Product Analysis JSON vendor_product = { - "top_vendors": { - "2023": top_vendors_2023.to_dict(orient='records'), - "2024": top_vendors_2024.to_dict(orient='records'), + "metadata": { + "description": "Vendor and product-level analysis of vulnerabilities for 2023 and 2024.", + "generated_on": generated_date, + "source": ["NVD", "CISA KEV"], + "attribution": { + "NVD": "This product uses the NVD API but is not endorsed or certified by the NVD.", + "CISA KEV": "Data from CISA KEV is used under the Creative Commons 0 1.0 License." + }, + "author": "2024 Vulnerability Insights Project" }, - "top_products": top_products, - "vendor_trends": vendor_trends, - "critical_spikes": critical_spikes, + "data": { + "top_vendors": { + "2023": top_vendors_2023.to_dict(orient='records'), + "2024": top_vendors_2024.to_dict(orient='records'), + }, + "top_products": top_products, + "vendor_trends": vendor_trends, + "critical_spikes": critical_spikes, + } } # Save the vendor_product metrics to a JSON file -with open("../../data/2024_insights/output/vendor_product.json", "w") as f: +with open("../../data/2024_insights/output/vendor_product_analysis.json", "w") as f: json.dump(vendor_product, f) vendor_product ``` -# CISA KEV Analysis - -This section evaluates the relationship between NVD data and the CISA KEV catalog: +## CISA KEV Analysis -1. **Vulnerabilities Added to CISA KEV**: Counts vulnerabilities added to the KEV catalog, with YoY comparison. -2. **CISA KEV Overlap with NVD**: Shows percentage of NVD vulnerabilities that are also in the KEV catalog. -3. **Top Exploited Vendors in CISA KEV**: Ranks vendors with the most KEV vulnerabilities. -4. **Time to CISA KEV Inclusion**: Calculates time between NVD publication and KEV inclusion. +The Known Exploited Vulnerabilities (KEV) catalog provides critical insights for organizations to prioritize patching based on active exploitation. This analysis includes: - - -# Specific CVE Details - -This section highlights vulnerabilities with high impact or severity: - -1. **Most Severe Vulnerabilities**: Lists CVEs with the highest CVSS scores. -2. **Most Impactful Vulnerabilities**: Combines multiple factors (CVSS, KEV inclusion, exploitation evidence) to rank vulnerabilities. +1. **Vulnerabilities Added to CISA KEV**: Tracks how many vulnerabilities were added to KEV for 2023 and 2024. +2. **CISA KEV Overlap with NVD**: Calculates the percentage of NVD vulnerabilities also present in the KEV catalog. +3. **Top Exploited Vendors in CISA KEV**: Identifies vendors most impacted by KEV vulnerabilities. +4. **Time to CISA KEV Inclusion**: Measures how quickly vulnerabilities transition from publication to KEV inclusion. ```python @@ -320,7 +354,7 @@ kev_records_2024 = df_2024[df_2024['CISA_KEV'] == True].copy() # Ensure datetime columns are in the correct format and remove timezone info for df in [kev_records_2023, kev_records_2024]: df['KEV_DateAdded'] = pd.to_datetime(df['KEV_DateAdded'], errors='coerce').dt.tz_localize(None) - df['Published_Date'] = pd.to_datetime(df['Published_Date'], errors='coerce').dt.tz_localize(None) +df['Published_Date'] = pd.to_datetime(df['Published_Date'], errors='coerce').dt.tz_localize(None) # Group CISA KEV Data by Month for 2023 and 2024 kev_additions = { @@ -376,12 +410,12 @@ top_kev_vendors = { vendor_rank_changes = [] for vendor in {v['KEV_Vendor'] for v in top_kev_vendors['2024']}: rank_2023 = next((i + 1 for i, v in enumerate(top_kev_vendors['2023']) if v['KEV_Vendor'] == vendor), None) - rank_2024 = next((i + 1 for i, v in enumerate(top_kev_vendors['2024']) if v['KEV_Vendor'] == vendor), None) - vendor_rank_changes.append({ - "vendor": vendor, - "2023_rank": rank_2023, - "2024_rank": rank_2024 - }) +rank_2024 = next((i + 1 for i, v in enumerate(top_kev_vendors['2024']) if v['KEV_Vendor'] == vendor), None) +vendor_rank_changes.append({ + "vendor": vendor, + "2023_rank": rank_2023, + "2024_rank": rank_2024 +}) # Sort the vendor_rank_changes by 2024 rank vendor_rank_changes = sorted(vendor_rank_changes, key=lambda x: x["2024_rank"] if x["2024_rank"] else float('inf')) @@ -390,39 +424,51 @@ vendor_rank_changes = sorted(vendor_rank_changes, key=lambda x: x["2024_rank"] i time_to_kev_inclusion = {} for year, records in {"2023": kev_records_2023, "2024": kev_records_2024}.items(): inclusion_times = records[records['KEV_DateAdded'].dt.year == int(year)].copy() - inclusion_times['Time_To_KEV'] = ( - inclusion_times['KEV_DateAdded'] - inclusion_times['Published_Date'] - ).dt.days - inclusion_times = inclusion_times[inclusion_times['Time_To_KEV'] >= 0] - time_to_kev_inclusion[year] = { - "min_days": int(inclusion_times['Time_To_KEV'].min()) if not inclusion_times.empty else None, - "max_days": int(inclusion_times['Time_To_KEV'].max()) if not inclusion_times.empty else None, - "average_days": round(float(inclusion_times['Time_To_KEV'].mean()), 2) if not inclusion_times.empty else None - } +inclusion_times['Time_To_KEV'] = ( + inclusion_times['KEV_DateAdded'] - inclusion_times['Published_Date'] +).dt.days +inclusion_times = inclusion_times[inclusion_times['Time_To_KEV'] >= 0] +time_to_kev_inclusion[year] = { + "min_days": int(inclusion_times['Time_To_KEV'].min()) if not inclusion_times.empty else None, + "max_days": int(inclusion_times['Time_To_KEV'].max()) if not inclusion_times.empty else None, + "average_days": round(float(inclusion_times['Time_To_KEV'].mean()), 2) if not inclusion_times.empty else None +} # Final JSON Structure cisa_kev = { - "kev_additions": kev_additions, - "kev_monthly_changes": kev_monthly_changes, - "nvd_kev_overlap": { - "data": kev_overlap, - "note": "Percentage of NVD vulnerabilities in a month that were also added to KEV." + "metadata": { + "description": "Analysis of CISA KEV catalog inclusion and overlap with NVD vulnerabilities for 2023 and 2024.", + "generated_on": generated_date, + "source": ["NVD", "CISA KEV"], + "attribution": { + "NVD": "This product uses the NVD API but is not endorsed or certified by the NVD.", + "CISA KEV": "Data from CISA KEV is used under the Creative Commons 0 1.0 License." + }, + "author": "2024 Vulnerability Insights Project" }, - "top_kev_vendors": top_kev_vendors, - "vendor_rank_changes": vendor_rank_changes, - "time_to_kev_inclusion": time_to_kev_inclusion + "data": { + "kev_additions": kev_additions, + "kev_monthly_changes": kev_monthly_changes, + "nvd_kev_overlap": { + "data": kev_overlap, + "note": "Percentage of NVD vulnerabilities in a month that were also added to KEV." + }, + "top_kev_vendors": top_kev_vendors, + "vendor_rank_changes": vendor_rank_changes, + "time_to_kev_inclusion": time_to_kev_inclusion + } } # Save the cisa_kev metrics to a JSON file -with open("../../data/2024_insights/output/cisa_kev.json", "w") as f: +with open("../../data/2024_insights/output/cisa_kev_analysis.json", "w") as f: json.dump(cisa_kev, f) cisa_kev ``` -# Specific CVE Details +## Specific CVE Details -This section highlights vulnerabilities with high impact or severity: +This section highlights vulnerabilities with high impact or severity to assist in prioritization and understanding of high-risk CVEs: 1. **Most Severe Vulnerabilities**: Lists CVEs with the highest CVSS scores. 2. **Most Impactful Vulnerabilities**: Combines multiple factors (CVSS, KEV inclusion, exploitation evidence) to rank vulnerabilities. @@ -441,9 +487,9 @@ if remaining_cves_needed > 0: .drop_duplicates() .head(remaining_cves_needed) ) - most_severe = pd.concat([cvss_10_cves, additional_cves]).drop_duplicates() +most_severe = pd.concat([cvss_10_cves, additional_cves]).drop_duplicates() else: - most_severe = cvss_10_cves +most_severe = cvss_10_cves # Ensure the final result is sorted and unique most_severe = ( @@ -458,6 +504,8 @@ df_2024['Exploitation_Evidence'] = df_2024['CVE_ID'].isin(kev_records_2024['CVE_ # Define Impact Score calculation function + + def calculate_impact_score(row): exploitation_weight = 10 if row['Exploitation_Evidence'] else 0 impact_score = row['CVSS_Base_Score'] * 2 + exploitation_weight @@ -478,27 +526,39 @@ most_impactful = ( # Final JSON structure specific_cve_details = { - "most_severe": most_severe, - "most_impactful": most_impactful, - "notes": { - "most_severe": "Includes all CVEs with CVSS_Base_Score of 10.0 and additional CVEs to make a total of 25, if fewer than 25 CVSS 10.0 CVEs exist.", - "most_impactful": "Top 10 CVEs by impact score, prioritizing exploitation evidence." + "metadata": { + "description": "Detailed analysis of the most severe and impactful CVEs for 2024.", + "generated_on": generated_date, + "source": ["NVD", "CISA KEV"], + "attribution": { + "NVD": "This product uses the NVD API but is not endorsed or certified by the NVD.", + "CISA KEV": "Data from CISA KEV is used under the Creative Commons 0 1.0 License." + }, + "author": "2024 Vulnerability Insights Project" + }, + "data": { + "most_severe": most_severe, + "most_impactful": most_impactful, + "notes": { + "most_severe": "Includes all CVEs with CVSS_Base_Score of 10.0 and additional CVEs to make a total of 25, if fewer than 25 CVSS 10.0 CVEs exist.", + "most_impactful": "Top 10 CVEs by impact score, prioritizing exploitation evidence." + } } } # Save the specific_cve_details metrics to a JSON file -with open("../../data/2024_insights/output/specific_cve_details.json", "w") as f: +with open("../../data/2024_insights/output/cve_details.json", "w") as f: json.dump(specific_cve_details, f) specific_cve_details ``` -# CVE Assigner Analysis - -This section identifies the organizations assigning the most CVEs: +## CVE Assigner Analysis -1. **Top CVE Assigners**: Highlights top assigners by the number of CVEs and severity breakdown (Critical and High). +This section identifies the organizations assigning the most CVEs to understand trends and priorities in vulnerability reporting. +1. **Top CVE Assigners**: Highlights top assigners by the number of CVEs and severity breakdown (Critical, High, Medium, Low). +2. **Year-over-Year Comparison**: Provides insights into changes in CVE assignment trends between 2023 and 2024. ```python # Group by CVE Assigner for 2023 and 2024 with severity breakdown and total counts @@ -547,19 +607,31 @@ percentage_changes = { # Transform to JSON-friendly structure top_assigners = { - "top_assigners": { - "2023": top_assigners_2023.to_dict(orient='records'), - "2024": top_assigners_2024.to_dict(orient='records'), - "comparison_notes": ( - "Critical: CVSS >= 9.0, High: 7.0 <= CVSS < 9.0, " - "Medium: 4.0 <= CVSS < 7.0, Low: CVSS < 4.0. " - "Shows top assigners for each year with severity breakdowns, " - "total counts, and year-over-year changes." - ), - "totals": { - "2023": total_2023, - "2024": total_2024, - "percentage_changes": percentage_changes + "metadata": { + "description": "Analysis of top CVE assigners and severity breakdowns for 2023 and 2024.", + "generated_on": generated_date, + "source": ["NVD", "CISA KEV"], + "attribution": { + "NVD": "This product uses the NVD API but is not endorsed or certified by the NVD.", + "CISA KEV": "Data from CISA KEV is used under the Creative Commons 0 1.0 License." + }, + "author": "2024 Vulnerability Insights Project" + }, + "data": { + "top_assigners": { + "2023": top_assigners_2023.to_dict(orient='records'), + "2024": top_assigners_2024.to_dict(orient='records'), + "comparison_notes": ( + "Critical: CVSS >= 9.0, High: 7.0 <= CVSS < 9.0, " + "Medium: 4.0 <= CVSS < 7.0, Low: CVSS < 4.0. " + "Shows top assigners for each year with severity breakdowns, " + "total counts, and year-over-year changes." + ), + "totals": { + "2023": total_2023, + "2024": total_2024, + "percentage_changes": percentage_changes + } } } } diff --git a/notebooks/2024_insights/03_analysis.ipynb b/notebooks/2024_insights/03_analysis.ipynb index ac02b49..6ec64b3 100644 --- a/notebooks/2024_insights/03_analysis.ipynb +++ b/notebooks/2024_insights/03_analysis.ipynb @@ -6,23 +6,37 @@ "metadata": { "collapsed": true, "ExecuteTime": { - "end_time": "2024-12-29T12:09:21.528409Z", - "start_time": "2024-12-29T12:09:21.522620Z" + "end_time": "2024-12-30T11:23:07.151396Z", + "start_time": "2024-12-30T11:23:04.257590Z" } }, "source": [ "import json\n", + "from datetime import datetime\n", "\n", - "import pandas as pd" + "import pandas as pd\n", + "\n", + "# Generate the current date\n", + "generated_date = datetime.now().strftime(\"%Y-%m-%d\")\n", + "\n", + "print(f\"Data generated on: {generated_date}\")" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Data generated on: 2024-12-30\n" + ] + } ], - "outputs": [], - "execution_count": 119 + "execution_count": 1 }, { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:23.080716Z", - "start_time": "2024-12-29T12:09:21.674312Z" + "end_time": "2024-12-30T11:23:09.262137Z", + "start_time": "2024-12-30T11:23:07.406596Z" } }, "cell_type": "code", @@ -250,32 +264,33 @@ "" ] }, - "execution_count": 120, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 120 + "execution_count": 2 }, { "metadata": {}, "cell_type": "markdown", "source": [ - "# Overview Metrics\n", + "## Overview Metrics\n", "\n", - "This section calculates high-level aggregate metrics to summarize vulnerability data:\n", + "This section summarizes high-level trends in vulnerability publication for 2023 and 2024 to provide a snapshot of key patterns.\n", "\n", - "1. **Total Vulnerabilities Published**: Total CVEs published in each year (2023 and 2024).\n", - "2. **Month with Most Vulnerabilities Published**: Identifies the month with the highest vulnerability count.\n", - "3. **Vulnerability Severity Distribution**: Calculates the distribution of Critical, High, Medium, and Low severity vulnerabilities.\n" + "Key insights calculated:\n", + "1. **Total Vulnerabilities Published**: How many CVEs were published each year and their percentage change.\n", + "2. **Month with Most Vulnerabilities Published**: Identifies peak publication months to understand seasonal patterns.\n", + "3. **Vulnerability Severity Distribution**: Examines the distribution of CVEs by severity (Critical, High, Medium, Low) for risk prioritization.\n" ], "id": "83b8c61e42636520" }, { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:23.161198Z", - "start_time": "2024-12-29T12:09:23.131421Z" + "end_time": "2024-12-30T11:23:09.409730Z", + "start_time": "2024-12-30T11:23:09.360943Z" } }, "cell_type": "code", @@ -319,9 +334,21 @@ "}\n", "\n", "overview_metrics = {\n", - " \"total_vulnerabilities\": total_vulnerabilities,\n", - " \"month_with_most_vulnerabilities\": month_with_most_vulnerabilities,\n", - " \"severity_distribution\": severity_distribution\n", + " \"metadata\": {\n", + " \"description\": \"Overview metrics summarizing vulnerability data for 2023 and 2024.\",\n", + " \"generated_on\": generated_date,\n", + " \"source\": [\"NVD\", \"CISA KEV\"],\n", + " \"attribution\": {\n", + " \"NVD\": \"This product uses the NVD API but is not endorsed or certified by the NVD.\",\n", + " \"CISA KEV\": \"Data from CISA KEV is used under the Creative Commons 0 1.0 License.\"\n", + " },\n", + " \"author\": \"2024 Vulnerability Insights Project\"\n", + " },\n", + " \"data\": {\n", + " \"total_vulnerabilities\": total_vulnerabilities,\n", + " \"month_with_most_vulnerabilities\": month_with_most_vulnerabilities,\n", + " \"severity_distribution\": severity_distribution\n", + " }\n", "}\n", "\n", "# Save the overview_metrics to a JSON file\n", @@ -335,52 +362,58 @@ { "data": { "text/plain": [ - "{'total_vulnerabilities': {'2023': 95665,\n", - " '2024': 85141,\n", - " 'percentage_change': -11.001},\n", - " 'month_with_most_vulnerabilities': {'2023': {'month': 10,\n", - " 'vulnerability_count': 10047},\n", - " '2024': {'month': 10, 'vulnerability_count': 13931},\n", - " 'comparison': {'percentage_change': 38.658,\n", - " 'difference_in_vulnerability_count': 3884}},\n", - " 'severity_distribution': {'2023': {'MEDIUM': 36592,\n", - " 'HIGH': 33515,\n", - " 'UNKNOWN': 13679,\n", - " 'CRITICAL': 9017,\n", - " 'LOW': 2862},\n", - " '2024': {'MEDIUM': 30752,\n", - " 'HIGH': 24674,\n", - " 'UNKNOWN': 23234,\n", - " 'CRITICAL': 4116,\n", - " 'LOW': 2365}}}" + "{'metadata': {'description': 'Overview metrics summarizing vulnerability data for 2023 and 2024.',\n", + " 'generated_on': '2024-12-30',\n", + " 'source': ['NVD', 'CISA KEV'],\n", + " 'attribution': {'NVD': 'This product uses the NVD API but is not endorsed or certified by the NVD.',\n", + " 'CISA KEV': 'Data from CISA KEV is used under the Creative Commons 0 1.0 License.'},\n", + " 'author': '2024 Vulnerability Insights Project'},\n", + " 'data': {'total_vulnerabilities': {'2023': 95665,\n", + " '2024': 85141,\n", + " 'percentage_change': -11.001},\n", + " 'month_with_most_vulnerabilities': {'2023': {'month': 10,\n", + " 'vulnerability_count': 10047},\n", + " '2024': {'month': 10, 'vulnerability_count': 13931},\n", + " 'comparison': {'percentage_change': 38.658,\n", + " 'difference_in_vulnerability_count': 3884}},\n", + " 'severity_distribution': {'2023': {'MEDIUM': 36592,\n", + " 'HIGH': 33515,\n", + " 'UNKNOWN': 13679,\n", + " 'CRITICAL': 9017,\n", + " 'LOW': 2862},\n", + " '2024': {'MEDIUM': 30752,\n", + " 'HIGH': 24674,\n", + " 'UNKNOWN': 23234,\n", + " 'CRITICAL': 4116,\n", + " 'LOW': 2365}}}}" ] }, - "execution_count": 121, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 121 + "execution_count": 3 }, { "metadata": {}, "cell_type": "markdown", "source": [ - "# Time-Series Metrics\n", + "## Time-Series Metrics\n", "\n", - "This section focuses on trends over time to analyze how vulnerabilities evolve month-to-month:\n", + "Understanding vulnerability trends over time helps identify patterns in CVE publication and potential shifts in the security landscape. This section analyzes:\n", "\n", - "1. **Monthly Vulnerability Counts**: Monthly counts of vulnerabilities published in 2023 and 2024.\n", - "2. **Severity Trends**: Tracks changes in vulnerability severities (Critical, High, Medium, Low) over time.\n", - "3. **Spike in Vulnerability Counts**: Highlights months with significant increases in vulnerability publication.\n" + "1. **Monthly Vulnerability Counts**: Compare month-to-month vulnerability counts in 2023 and 2024.\n", + "2. **Severity Trends**: Examine how vulnerability severities evolved over time.\n", + "3. **Spike in Vulnerability Counts**: Highlight months with sharp increases to correlate with external events or publication cycles.\n" ], "id": "6746a9477a5f2c09" }, { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:23.270272Z", - "start_time": "2024-12-29T12:09:23.219455Z" + "end_time": "2024-12-30T11:23:09.713919Z", + "start_time": "2024-12-30T11:23:09.566462Z" } }, "cell_type": "code", @@ -435,9 +468,21 @@ " }\n", "}\n", "time_series_metrics = {\n", - " \"monthly_vulnerability_counts\": monthly_vulnerability_counts,\n", - " \"severity_trends\": severity_trends,\n", - " \"spike_analysis\": spike_analysis,\n", + " \"metadata\": {\n", + " \"description\": \"Time-series analysis of monthly vulnerability trends for 2023 and 2024.\",\n", + " \"generated_on\": generated_date,\n", + " \"source\": [\"NVD\", \"CISA KEV\"],\n", + " \"attribution\": {\n", + " \"NVD\": \"This product uses the NVD API but is not endorsed or certified by the NVD.\",\n", + " \"CISA KEV\": \"Data from CISA KEV is used under the Creative Commons 0 1.0 License.\"\n", + " },\n", + " \"author\": \"2024 Vulnerability Insights Project\"\n", + " },\n", + " \"data\": {\n", + " \"monthly_vulnerability_counts\": monthly_vulnerability_counts,\n", + " \"severity_trends\": severity_trends,\n", + " \"spike_analysis\": spike_analysis,\n", + " }\n", "}\n", "\n", "# Save the time_series_metrics to a JSON file\n", @@ -451,182 +496,188 @@ { "data": { "text/plain": [ - "{'monthly_vulnerability_counts': {'2023': {1: 8860,\n", - " 2: 8610,\n", - " 3: 7029,\n", - " 4: 7144,\n", - " 5: 6734,\n", - " 6: 6085,\n", - " 7: 9350,\n", - " 8: 7168,\n", - " 9: 9176,\n", - " 10: 10047,\n", - " 11: 9114,\n", - " 12: 6348},\n", - " '2024': {1: 4557,\n", - " 2: 4873,\n", - " 3: 3979,\n", - " 4: 5133,\n", - " 5: 4583,\n", - " 6: 5723,\n", - " 7: 9795,\n", - " 8: 9152,\n", - " 9: 11683,\n", - " 10: 13931,\n", - " 11: 8403,\n", - " 12: 3329}},\n", - " 'severity_trends': {'2023': {1: {'CRITICAL': 420,\n", - " 'HIGH': 2555,\n", - " 'MEDIUM': 3272,\n", - " 'LOW': 66,\n", - " 'UNKNOWN': 2547},\n", - " 2: {'CRITICAL': 557,\n", - " 'HIGH': 2988,\n", - " 'MEDIUM': 3855,\n", - " 'LOW': 556,\n", - " 'UNKNOWN': 654},\n", - " 3: {'CRITICAL': 552,\n", - " 'HIGH': 2778,\n", - " 'MEDIUM': 2756,\n", - " 'LOW': 187,\n", - " 'UNKNOWN': 756},\n", - " 4: {'CRITICAL': 568,\n", - " 'HIGH': 1937,\n", - " 'MEDIUM': 1998,\n", - " 'LOW': 237,\n", - " 'UNKNOWN': 2404},\n", - " 5: {'CRITICAL': 470,\n", - " 'HIGH': 1811,\n", - " 'MEDIUM': 2930,\n", - " 'LOW': 122,\n", - " 'UNKNOWN': 1401},\n", - " 6: {'CRITICAL': 621,\n", - " 'HIGH': 2189,\n", - " 'MEDIUM': 2186,\n", - " 'LOW': 130,\n", - " 'UNKNOWN': 959},\n", - " 7: {'CRITICAL': 783,\n", - " 'HIGH': 3502,\n", - " 'MEDIUM': 3670,\n", - " 'LOW': 358,\n", - " 'UNKNOWN': 1037},\n", - " 8: {'CRITICAL': 794,\n", - " 'HIGH': 2911,\n", - " 'MEDIUM': 2499,\n", - " 'LOW': 216,\n", - " 'UNKNOWN': 748},\n", - " 9: {'CRITICAL': 1967,\n", - " 'HIGH': 2280,\n", - " 'MEDIUM': 3953,\n", - " 'LOW': 462,\n", - " 'UNKNOWN': 514},\n", - " 10: {'CRITICAL': 729,\n", - " 'HIGH': 4040,\n", - " 'MEDIUM': 3622,\n", - " 'LOW': 141,\n", - " 'UNKNOWN': 1515},\n", - " 11: {'CRITICAL': 911,\n", - " 'HIGH': 4268,\n", - " 'MEDIUM': 3331,\n", - " 'LOW': 317,\n", - " 'UNKNOWN': 287},\n", - " 12: {'CRITICAL': 645,\n", - " 'HIGH': 2256,\n", - " 'MEDIUM': 2520,\n", - " 'LOW': 70,\n", - " 'UNKNOWN': 857}},\n", - " '2024': {1: {'CRITICAL': 629,\n", - " 'HIGH': 1465,\n", - " 'MEDIUM': 2213,\n", - " 'LOW': 43,\n", - " 'UNKNOWN': 207},\n", - " 2: {'CRITICAL': 275,\n", - " 'HIGH': 1948,\n", - " 'MEDIUM': 1601,\n", - " 'LOW': 158,\n", - " 'UNKNOWN': 891},\n", - " 3: {'CRITICAL': 52,\n", - " 'HIGH': 639,\n", - " 'MEDIUM': 469,\n", - " 'LOW': 139,\n", - " 'UNKNOWN': 2680},\n", - " 4: {'CRITICAL': 215,\n", - " 'HIGH': 759,\n", - " 'MEDIUM': 930,\n", - " 'LOW': 83,\n", - " 'UNKNOWN': 3146},\n", - " 5: {'CRITICAL': 49, 'HIGH': 699, 'MEDIUM': 741, 'LOW': 7, 'UNKNOWN': 3087},\n", - " 6: {'CRITICAL': 439,\n", - " 'HIGH': 1123,\n", - " 'MEDIUM': 1729,\n", - " 'LOW': 159,\n", - " 'UNKNOWN': 2273},\n", - " 7: {'CRITICAL': 435,\n", - " 'HIGH': 3170,\n", - " 'MEDIUM': 4153,\n", - " 'LOW': 360,\n", - " 'UNKNOWN': 1677},\n", - " 8: {'CRITICAL': 530,\n", - " 'HIGH': 3065,\n", - " 'MEDIUM': 3515,\n", - " 'LOW': 188,\n", - " 'UNKNOWN': 1854},\n", - " 9: {'CRITICAL': 586,\n", - " 'HIGH': 5690,\n", - " 'MEDIUM': 4295,\n", - " 'LOW': 421,\n", - " 'UNKNOWN': 691},\n", - " 10: {'CRITICAL': 548,\n", - " 'HIGH': 3689,\n", - " 'MEDIUM': 7208,\n", - " 'LOW': 623,\n", - " 'UNKNOWN': 1863},\n", - " 11: {'CRITICAL': 249,\n", - " 'HIGH': 2104,\n", - " 'MEDIUM': 2905,\n", - " 'LOW': 170,\n", - " 'UNKNOWN': 2975},\n", - " 12: {'CRITICAL': 109,\n", - " 'HIGH': 323,\n", - " 'MEDIUM': 993,\n", - " 'LOW': 14,\n", - " 'UNKNOWN': 1890}}},\n", - " 'spike_analysis': {'2023': {7: {'percentage_increase': 53.66,\n", - " 'vulnerability_count': 9350},\n", - " 9: {'percentage_increase': 28.01, 'vulnerability_count': 9176}},\n", - " '2024': {4: {'percentage_increase': 29.0, 'vulnerability_count': 5133},\n", - " 6: {'percentage_increase': 24.87, 'vulnerability_count': 5723},\n", - " 7: {'percentage_increase': 71.15, 'vulnerability_count': 9795},\n", - " 9: {'percentage_increase': 27.66, 'vulnerability_count': 11683}}}}" + "{'metadata': {'description': 'Time-series analysis of monthly vulnerability trends for 2023 and 2024.',\n", + " 'generated_on': '2024-12-30',\n", + " 'source': ['NVD', 'CISA KEV'],\n", + " 'attribution': {'NVD': 'This product uses the NVD API but is not endorsed or certified by the NVD.',\n", + " 'CISA KEV': 'Data from CISA KEV is used under the Creative Commons 0 1.0 License.'},\n", + " 'author': '2024 Vulnerability Insights Project'},\n", + " 'data': {'monthly_vulnerability_counts': {'2023': {1: 8860,\n", + " 2: 8610,\n", + " 3: 7029,\n", + " 4: 7144,\n", + " 5: 6734,\n", + " 6: 6085,\n", + " 7: 9350,\n", + " 8: 7168,\n", + " 9: 9176,\n", + " 10: 10047,\n", + " 11: 9114,\n", + " 12: 6348},\n", + " '2024': {1: 4557,\n", + " 2: 4873,\n", + " 3: 3979,\n", + " 4: 5133,\n", + " 5: 4583,\n", + " 6: 5723,\n", + " 7: 9795,\n", + " 8: 9152,\n", + " 9: 11683,\n", + " 10: 13931,\n", + " 11: 8403,\n", + " 12: 3329}},\n", + " 'severity_trends': {'2023': {1: {'CRITICAL': 420,\n", + " 'HIGH': 2555,\n", + " 'MEDIUM': 3272,\n", + " 'LOW': 66,\n", + " 'UNKNOWN': 2547},\n", + " 2: {'CRITICAL': 557,\n", + " 'HIGH': 2988,\n", + " 'MEDIUM': 3855,\n", + " 'LOW': 556,\n", + " 'UNKNOWN': 654},\n", + " 3: {'CRITICAL': 552,\n", + " 'HIGH': 2778,\n", + " 'MEDIUM': 2756,\n", + " 'LOW': 187,\n", + " 'UNKNOWN': 756},\n", + " 4: {'CRITICAL': 568,\n", + " 'HIGH': 1937,\n", + " 'MEDIUM': 1998,\n", + " 'LOW': 237,\n", + " 'UNKNOWN': 2404},\n", + " 5: {'CRITICAL': 470,\n", + " 'HIGH': 1811,\n", + " 'MEDIUM': 2930,\n", + " 'LOW': 122,\n", + " 'UNKNOWN': 1401},\n", + " 6: {'CRITICAL': 621,\n", + " 'HIGH': 2189,\n", + " 'MEDIUM': 2186,\n", + " 'LOW': 130,\n", + " 'UNKNOWN': 959},\n", + " 7: {'CRITICAL': 783,\n", + " 'HIGH': 3502,\n", + " 'MEDIUM': 3670,\n", + " 'LOW': 358,\n", + " 'UNKNOWN': 1037},\n", + " 8: {'CRITICAL': 794,\n", + " 'HIGH': 2911,\n", + " 'MEDIUM': 2499,\n", + " 'LOW': 216,\n", + " 'UNKNOWN': 748},\n", + " 9: {'CRITICAL': 1967,\n", + " 'HIGH': 2280,\n", + " 'MEDIUM': 3953,\n", + " 'LOW': 462,\n", + " 'UNKNOWN': 514},\n", + " 10: {'CRITICAL': 729,\n", + " 'HIGH': 4040,\n", + " 'MEDIUM': 3622,\n", + " 'LOW': 141,\n", + " 'UNKNOWN': 1515},\n", + " 11: {'CRITICAL': 911,\n", + " 'HIGH': 4268,\n", + " 'MEDIUM': 3331,\n", + " 'LOW': 317,\n", + " 'UNKNOWN': 287},\n", + " 12: {'CRITICAL': 645,\n", + " 'HIGH': 2256,\n", + " 'MEDIUM': 2520,\n", + " 'LOW': 70,\n", + " 'UNKNOWN': 857}},\n", + " '2024': {1: {'CRITICAL': 629,\n", + " 'HIGH': 1465,\n", + " 'MEDIUM': 2213,\n", + " 'LOW': 43,\n", + " 'UNKNOWN': 207},\n", + " 2: {'CRITICAL': 275,\n", + " 'HIGH': 1948,\n", + " 'MEDIUM': 1601,\n", + " 'LOW': 158,\n", + " 'UNKNOWN': 891},\n", + " 3: {'CRITICAL': 52,\n", + " 'HIGH': 639,\n", + " 'MEDIUM': 469,\n", + " 'LOW': 139,\n", + " 'UNKNOWN': 2680},\n", + " 4: {'CRITICAL': 215,\n", + " 'HIGH': 759,\n", + " 'MEDIUM': 930,\n", + " 'LOW': 83,\n", + " 'UNKNOWN': 3146},\n", + " 5: {'CRITICAL': 49, 'HIGH': 699, 'MEDIUM': 741, 'LOW': 7, 'UNKNOWN': 3087},\n", + " 6: {'CRITICAL': 439,\n", + " 'HIGH': 1123,\n", + " 'MEDIUM': 1729,\n", + " 'LOW': 159,\n", + " 'UNKNOWN': 2273},\n", + " 7: {'CRITICAL': 435,\n", + " 'HIGH': 3170,\n", + " 'MEDIUM': 4153,\n", + " 'LOW': 360,\n", + " 'UNKNOWN': 1677},\n", + " 8: {'CRITICAL': 530,\n", + " 'HIGH': 3065,\n", + " 'MEDIUM': 3515,\n", + " 'LOW': 188,\n", + " 'UNKNOWN': 1854},\n", + " 9: {'CRITICAL': 586,\n", + " 'HIGH': 5690,\n", + " 'MEDIUM': 4295,\n", + " 'LOW': 421,\n", + " 'UNKNOWN': 691},\n", + " 10: {'CRITICAL': 548,\n", + " 'HIGH': 3689,\n", + " 'MEDIUM': 7208,\n", + " 'LOW': 623,\n", + " 'UNKNOWN': 1863},\n", + " 11: {'CRITICAL': 249,\n", + " 'HIGH': 2104,\n", + " 'MEDIUM': 2905,\n", + " 'LOW': 170,\n", + " 'UNKNOWN': 2975},\n", + " 12: {'CRITICAL': 109,\n", + " 'HIGH': 323,\n", + " 'MEDIUM': 993,\n", + " 'LOW': 14,\n", + " 'UNKNOWN': 1890}}},\n", + " 'spike_analysis': {'2023': {7: {'percentage_increase': 53.66,\n", + " 'vulnerability_count': 9350},\n", + " 9: {'percentage_increase': 28.01, 'vulnerability_count': 9176}},\n", + " '2024': {4: {'percentage_increase': 29.0, 'vulnerability_count': 5133},\n", + " 6: {'percentage_increase': 24.87, 'vulnerability_count': 5723},\n", + " 7: {'percentage_increase': 71.15, 'vulnerability_count': 9795},\n", + " 9: {'percentage_increase': 27.66, 'vulnerability_count': 11683}}}}}" ] }, - "execution_count": 122, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 122 + "execution_count": 4 }, { "metadata": {}, "cell_type": "markdown", "source": [ - "# Vendor/Product Analysis\n", + "## Vendor/Product Analysis\n", "\n", - "This section investigates vulnerabilities at the vendor and product levels:\n", + "Analyzing vulnerabilities by vendor and product provides actionable insights for vendor risk management and prioritizing patches. Key metrics include:\n", "\n", - "1. **Top Vendors by Vulnerabilities**: Ranks vendors with the most vulnerabilities, focusing on Critical and High severities.\n", - "2. **Top Products by Vulnerabilities**: Visualizes vulnerabilities grouped by product, nested under their vendors.\n", - "3. **Vulnerability Trends by Vendor**: Tracks monthly trends for top vendors across 2023 and 2024.\n", - "4. **Critical Vulnerability Spike Analysis**: Explores Critical vulnerabilities by vendor and product for peak months.\n" + "1. **Top Vendors by Vulnerabilities**: Identify vendors with the most vulnerabilities, focusing on high-severity issues.\n", + "2. **Top Products by Vulnerabilities**: Drill down into specific products with the highest vulnerability counts.\n", + "3. **Vulnerability Trends by Vendor**: Explore how vulnerabilities evolve for major vendors over time.\n", + "4. **Critical Vulnerability Spike Analysis**: Detect peak months for critical vulnerabilities by vendor and product to focus remediation efforts.\n" ], "id": "df9d78b729c56fd4" }, { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:23.874707Z", - "start_time": "2024-12-29T12:09:23.323534Z" + "end_time": "2024-12-30T11:23:11.988817Z", + "start_time": "2024-12-30T11:23:10.460422Z" } }, "cell_type": "code", @@ -703,53 +754,65 @@ "\n", "for vendor in top_vendors_2024['Vendor']:\n", " critical_spikes[\"2024\"][vendor] = {}\n", - " vendor_critical = critical_vulns_2024[critical_vulns_2024['Vendor'] == vendor]\n", - " for product in top_products[\"2024\"].get(vendor, []):\n", - " product_name = product['Product']\n", - " product_critical = vendor_critical[vendor_critical['Product'] == product_name]\n", - " if not product_critical.empty:\n", - " top_month = (\n", - " product_critical.groupby('Published_Month').size().idxmax()\n", - " )\n", - " count = (\n", - " product_critical.groupby('Published_Month').size().max()\n", - " )\n", - " critical_spikes[\"2024\"][vendor][product_name] = {\n", - " \"month\": pd.to_datetime(f\"2024-{top_month}-01\").strftime('%B'),\n", - " \"count\": int(count)\n", - " }\n", + "vendor_critical = critical_vulns_2024[critical_vulns_2024['Vendor'] == vendor]\n", + "for product in top_products[\"2024\"].get(vendor, []):\n", + " product_name = product['Product']\n", + "product_critical = vendor_critical[vendor_critical['Product'] == product_name]\n", + "if not product_critical.empty:\n", + " top_month = (\n", + " product_critical.groupby('Published_Month').size().idxmax()\n", + " )\n", + "count = (\n", + " product_critical.groupby('Published_Month').size().max()\n", + ")\n", + "critical_spikes[\"2024\"][vendor][product_name] = {\n", + " \"month\": pd.to_datetime(f\"2024-{top_month}-01\").strftime('%B'),\n", + " \"count\": int(count)\n", + "}\n", "\n", "for vendor in top_vendors_2023['Vendor']:\n", " critical_spikes[\"2023\"][vendor] = {}\n", - " vendor_critical = critical_vulns_2023[critical_vulns_2023['Vendor'] == vendor]\n", - " for product in top_products[\"2023\"].get(vendor, []):\n", - " product_name = product['Product']\n", - " product_critical = vendor_critical[vendor_critical['Product'] == product_name]\n", - " if not product_critical.empty:\n", - " top_month = (\n", - " product_critical.groupby('Published_Month').size().idxmax()\n", - " )\n", - " count = (\n", - " product_critical.groupby('Published_Month').size().max()\n", - " )\n", - " critical_spikes[\"2023\"][vendor][product_name] = {\n", - " \"month\": pd.to_datetime(f\"2023-{top_month}-01\").strftime('%B'),\n", - " \"count\": int(count)\n", - " }\n", + "vendor_critical = critical_vulns_2023[critical_vulns_2023['Vendor'] == vendor]\n", + "for product in top_products[\"2023\"].get(vendor, []):\n", + " product_name = product['Product']\n", + "product_critical = vendor_critical[vendor_critical['Product'] == product_name]\n", + "if not product_critical.empty:\n", + " top_month = (\n", + " product_critical.groupby('Published_Month').size().idxmax()\n", + " )\n", + "count = (\n", + " product_critical.groupby('Published_Month').size().max()\n", + ")\n", + "critical_spikes[\"2023\"][vendor][product_name] = {\n", + " \"month\": pd.to_datetime(f\"2023-{top_month}-01\").strftime('%B'),\n", + " \"count\": int(count)\n", + "}\n", "\n", "# Final Vendor/Product Analysis JSON\n", "vendor_product = {\n", - " \"top_vendors\": {\n", - " \"2023\": top_vendors_2023.to_dict(orient='records'),\n", - " \"2024\": top_vendors_2024.to_dict(orient='records'),\n", + " \"metadata\": {\n", + " \"description\": \"Vendor and product-level analysis of vulnerabilities for 2023 and 2024.\",\n", + " \"generated_on\": generated_date,\n", + " \"source\": [\"NVD\", \"CISA KEV\"],\n", + " \"attribution\": {\n", + " \"NVD\": \"This product uses the NVD API but is not endorsed or certified by the NVD.\",\n", + " \"CISA KEV\": \"Data from CISA KEV is used under the Creative Commons 0 1.0 License.\"\n", + " },\n", + " \"author\": \"2024 Vulnerability Insights Project\"\n", " },\n", - " \"top_products\": top_products,\n", - " \"vendor_trends\": vendor_trends,\n", - " \"critical_spikes\": critical_spikes,\n", + " \"data\": {\n", + " \"top_vendors\": {\n", + " \"2023\": top_vendors_2023.to_dict(orient='records'),\n", + " \"2024\": top_vendors_2024.to_dict(orient='records'),\n", + " },\n", + " \"top_products\": top_products,\n", + " \"vendor_trends\": vendor_trends,\n", + " \"critical_spikes\": critical_spikes,\n", + " }\n", "}\n", "\n", "# Save the vendor_product metrics to a JSON file\n", - "with open(\"../../data/2024_insights/output/vendor_product.json\", \"w\") as f:\n", + "with open(\"../../data/2024_insights/output/vendor_product_analysis.json\", \"w\") as f:\n", " json.dump(vendor_product, f)\n", "\n", "vendor_product" @@ -757,231 +820,38 @@ "id": "82d6e7106cd5a47f", "outputs": [ { - "data": { - "text/plain": [ - "{'top_vendors': {'2023': [{'Vendor': 'Microsoft', 'count': 9748},\n", - " {'Vendor': 'Samsung', 'count': 8268},\n", - " {'Vendor': 'Cisco', 'count': 6286},\n", - " {'Vendor': 'Juniper', 'count': 4867},\n", - " {'Vendor': 'Discourse', 'count': 4751},\n", - " {'Vendor': 'F5', 'count': 2189},\n", - " {'Vendor': 'Apple', 'count': 2000},\n", - " {'Vendor': 'Adobe', 'count': 1897},\n", - " {'Vendor': 'Linux', 'count': 1786},\n", - " {'Vendor': 'Checkmk', 'count': 1584}],\n", - " '2024': [{'Vendor': 'Cisco', 'count': 8942},\n", - " {'Vendor': 'Microsoft', 'count': 8066},\n", - " {'Vendor': 'Linux', 'count': 8055},\n", - " {'Vendor': 'Samsung', 'count': 7756},\n", - " {'Vendor': 'Adobe', 'count': 5516},\n", - " {'Vendor': 'Checkmk', 'count': 2382},\n", - " {'Vendor': 'Apple', 'count': 1776},\n", - " {'Vendor': 'Juniper', 'count': 1656},\n", - " {'Vendor': 'Liferay', 'count': 1022},\n", - " {'Vendor': 'Ivanti', 'count': 1010}]},\n", - " 'top_products': {'2023': {'Microsoft': [{'Product': 'Windows_Server_2012',\n", - " 'count': 863},\n", - " {'Product': 'Windows_Server_2008', 'count': 779},\n", - " {'Product': 'Windows_10_22H2', 'count': 717},\n", - " {'Product': 'Windows_10_1809', 'count': 712},\n", - " {'Product': 'Windows_10_21H2', 'count': 691}],\n", - " 'Samsung': [{'Product': 'Android', 'count': 8196},\n", - " {'Product': 'Galaxy_Store', 'count': 8},\n", - " {'Product': 'Account', 'count': 7},\n", - " {'Product': 'Pass', 'count': 6},\n", - " {'Product': 'Samsung_Blockchain_Keystore', 'count': 6}],\n", - " 'Cisco': [{'Product': 'Ios', 'count': 1777},\n", - " {'Product': 'Adaptive_Security_Appliance_Software', 'count': 1287},\n", - " {'Product': 'Ios_Xe', 'count': 1182},\n", - " {'Product': 'Firepower_Threat_Defense', 'count': 874},\n", - " {'Product': 'Identity_Services_Engine', 'count': 319}],\n", - " 'Juniper': [{'Product': 'Junos', 'count': 3343},\n", - " {'Product': 'Junos_Os_Evolved', 'count': 1523},\n", - " {'Product': 'Paragon_Active_Assurance', 'count': 1}],\n", - " 'Discourse': [{'Product': 'Discourse', 'count': 4745},\n", - " {'Product': 'Discourse-Encrypt', 'count': 1},\n", - " {'Product': 'Discourse_Calendar', 'count': 1},\n", - " {'Product': 'Discourse_Jira', 'count': 1},\n", - " {'Product': 'Discourse_Reactions', 'count': 1}],\n", - " 'F5': [{'Product': 'Big-Ip_Access_Policy_Manager', 'count': 156},\n", - " {'Product': 'Big-Ip_Local_Traffic_Manager', 'count': 137},\n", - " {'Product': 'Big-Ip_Domain_Name_System', 'count': 132},\n", - " {'Product': 'Big-Ip_Application_Security_Manager', 'count': 129},\n", - " {'Product': 'Big-Ip_Advanced_Firewall_Manager', 'count': 129}],\n", - " 'Apple': [{'Product': 'Macos', 'count': 805},\n", - " {'Product': 'Iphone_Os', 'count': 409},\n", - " {'Product': 'Ipados', 'count': 389},\n", - " {'Product': 'Watchos', 'count': 174},\n", - " {'Product': 'Tvos', 'count': 143}],\n", - " 'Adobe': [{'Product': 'Commerce', 'count': 789},\n", - " {'Product': 'Coldfusion', 'count': 373},\n", - " {'Product': 'Experience_Manager', 'count': 301},\n", - " {'Product': 'Magento', 'count': 213},\n", - " {'Product': 'Experience_Manager_Cloud_Service', 'count': 136}],\n", - " 'Linux': [{'Product': 'Linux_Kernel', 'count': 1786}],\n", - " 'Checkmk': [{'Product': 'Checkmk', 'count': 1584}]},\n", - " '2024': {'Cisco': [{'Product': 'Ios', 'count': 2071},\n", - " {'Product': 'Ios_Xe', 'count': 1718},\n", - " {'Product': 'Adaptive_Security_Appliance_Software', 'count': 1570},\n", - " {'Product': 'Secure_Firewall_Management_Center', 'count': 1139},\n", - " {'Product': 'Firepower_Threat_Defense_Software', 'count': 540}],\n", - " 'Microsoft': [{'Product': 'Windows_Server_2012', 'count': 596},\n", - " {'Product': 'Windows_10_22H2', 'count': 556},\n", - " {'Product': 'Windows_Server_2008', 'count': 542},\n", - " {'Product': 'Windows_11_23H2', 'count': 516},\n", - " {'Product': 'Windows_10_1809', 'count': 504}],\n", - " 'Linux': [{'Product': 'Linux_Kernel', 'count': 8055}],\n", - " 'Samsung': [{'Product': 'Android', 'count': 7710},\n", - " {'Product': 'Notes', 'count': 19},\n", - " {'Product': 'Galaxy_Store', 'count': 4},\n", - " {'Product': 'Internet', 'count': 3},\n", - " {'Product': 'Email', 'count': 2}],\n", - " 'Adobe': [{'Product': 'Commerce', 'count': 2716},\n", - " {'Product': 'Magento', 'count': 1719},\n", - " {'Product': 'Experience_Manager', 'count': 622},\n", - " {'Product': 'Commerce_B2B', 'count': 198},\n", - " {'Product': 'Coldfusion', 'count': 108}],\n", - " 'Checkmk': [{'Product': 'Checkmk', 'count': 2382}],\n", - " 'Apple': [{'Product': 'Macos', 'count': 674},\n", - " {'Product': 'Iphone_Os', 'count': 330},\n", - " {'Product': 'Ipados', 'count': 319},\n", - " {'Product': 'Watchos', 'count': 139},\n", - " {'Product': 'Tvos', 'count': 123}],\n", - " 'Juniper': [{'Product': 'Junos', 'count': 996},\n", - " {'Product': 'Junos_Os_Evolved', 'count': 654},\n", - " {'Product': 'Paragon_Active_Assurance_Control_Center', 'count': 6}],\n", - " 'Liferay': [{'Product': 'Digital_Experience_Platform', 'count': 866},\n", - " {'Product': 'Dxp', 'count': 127},\n", - " {'Product': 'Liferay_Portal', 'count': 29}],\n", - " 'Ivanti': [{'Product': 'Connect_Secure', 'count': 324},\n", - " {'Product': 'Policy_Secure', 'count': 269},\n", - " {'Product': 'Endpoint_Manager', 'count': 246},\n", - " {'Product': 'Avalanche', 'count': 111},\n", - " {'Product': 'Neurons_For_Zero-Trust_Access', 'count': 12}]}},\n", - " 'vendor_trends': {'2023': {'Microsoft': [1674,\n", - " 819,\n", - " 763,\n", - " 941,\n", - " 318,\n", - " 812,\n", - " 1212,\n", - " 683,\n", - " 414,\n", - " 1184,\n", - " 651,\n", - " 277],\n", - " 'Samsung': [0, 1807, 387, 0, 707, 116, 1507, 1189, 1112, 309, 885, 249],\n", - " 'Cisco': [149, 9, 1283, 55, 112, 63, 65, 102, 2602, 144, 1463, 239],\n", - " 'Juniper': [1013, 0, 0, 1141, 0, 176, 426, 0, 192, 1919, 0, 0],\n", - " 'Discourse': [1478, 210, 434, 225, 0, 20, 1692, 0, 652, 14, 26, 0],\n", - " 'F5': [0, 469, 2, 4, 336, 0, 0, 350, 12, 1003, 13, 0],\n", - " 'Apple': [167, 234, 68, 13, 257, 310, 304, 11, 362, 156, 16, 102],\n", - " 'Adobe': [4, 2, 174, 2, 14, 432, 103, 37, 95, 522, 115, 397],\n", - " 'Linux': [112, 277, 370, 150, 115, 121, 158, 194, 83, 84, 79, 43],\n", - " 'Checkmk': [421, 0, 76, 165, 219, 14, 0, 207, 0, 0, 475, 7]},\n", - " '2024': {'Cisco': [230, 259, 0, 839, 1, 20, 0, 97, 4293, 3023, 180, 0],\n", - " 'Microsoft': [638, 678, 659, 207, 112, 472, 1271, 841, 811, 1318, 1031, 28],\n", - " 'Linux': [83, 144, 38, 145, 493, 336, 927, 573, 1702, 1468, 1882, 264],\n", - " 'Samsung': [327, 940, 2, 0, 0, 0, 2100, 1548, 1320, 627, 892, 0],\n", - " 'Adobe': [2, 95, 108, 28, 5, 1058, 8, 1464, 46, 2439, 29, 234],\n", - " 'Checkmk': [0, 0, 261, 136, 100, 365, 726, 235, 128, 356, 75, 0],\n", - " 'Apple': [92, 0, 292, 16, 90, 199, 333, 0, 212, 289, 43, 210],\n", - " 'Juniper': [1175, 0, 0, 0, 0, 0, 481, 0, 0, 0, 0, 0],\n", - " 'Liferay': [0, 488, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0],\n", - " 'Ivanti': [305, 8, 0, 240, 71, 0, 0, 117, 131, 10, 125, 3]}},\n", - " 'critical_spikes': {'2023': {'Microsoft': {'Windows_Server_2012': {'month': 'July',\n", - " 'count': 10},\n", - " 'Windows_Server_2008': {'month': 'February', 'count': 10},\n", - " 'Windows_10_22H2': {'month': 'February', 'count': 10},\n", - " 'Windows_10_1809': {'month': 'February', 'count': 10},\n", - " 'Windows_10_21H2': {'month': 'February', 'count': 10}},\n", - " 'Samsung': {'Android': {'month': 'August', 'count': 76},\n", - " 'Galaxy_Store': {'month': 'May', 'count': 1}},\n", - " 'Cisco': {'Ios': {'month': 'September', 'count': 615},\n", - " 'Adaptive_Security_Appliance_Software': {'month': 'September',\n", - " 'count': 246},\n", - " 'Ios_Xe': {'month': 'September', 'count': 381},\n", - " 'Firepower_Threat_Defense': {'month': 'September', 'count': 245}},\n", - " 'Juniper': {'Junos': {'month': 'April', 'count': 138}},\n", - " 'Discourse': {'Discourse': {'month': 'November', 'count': 4}},\n", - " 'F5': {'Big-Ip_Access_Policy_Manager': {'month': 'October', 'count': 5},\n", - " 'Big-Ip_Local_Traffic_Manager': {'month': 'October', 'count': 5},\n", - " 'Big-Ip_Domain_Name_System': {'month': 'October', 'count': 5},\n", - " 'Big-Ip_Application_Security_Manager': {'month': 'October', 'count': 5},\n", - " 'Big-Ip_Advanced_Firewall_Manager': {'month': 'October', 'count': 5}},\n", - " 'Apple': {'Macos': {'month': 'July', 'count': 14},\n", - " 'Iphone_Os': {'month': 'July', 'count': 9},\n", - " 'Ipados': {'month': 'July', 'count': 9},\n", - " 'Watchos': {'month': 'July', 'count': 4},\n", - " 'Tvos': {'month': 'July', 'count': 3}},\n", - " 'Adobe': {},\n", - " 'Linux': {'Linux_Kernel': {'month': 'July', 'count': 21}},\n", - " 'Checkmk': {}},\n", - " '2024': {'Cisco': {'Ios_Xe': {'month': 'September', 'count': 197},\n", - " 'Secure_Firewall_Management_Center': {'month': 'October', 'count': 92}},\n", - " 'Microsoft': {'Windows_Server_2012': {'month': 'August', 'count': 6},\n", - " 'Windows_10_22H2': {'month': 'September', 'count': 4},\n", - " 'Windows_Server_2008': {'month': 'August', 'count': 6},\n", - " 'Windows_11_23H2': {'month': 'August', 'count': 3},\n", - " 'Windows_10_1809': {'month': 'August', 'count': 3}},\n", - " 'Linux': {'Linux_Kernel': {'month': 'October', 'count': 6}},\n", - " 'Samsung': {'Notes': {'month': 'September', 'count': 1}},\n", - " 'Adobe': {'Commerce': {'month': 'June', 'count': 46},\n", - " 'Magento': {'month': 'June', 'count': 24},\n", - " 'Experience_Manager': {'month': 'June', 'count': 2}},\n", - " 'Checkmk': {'Checkmk': {'month': 'April', 'count': 92}},\n", - " 'Apple': {'Macos': {'month': 'December', 'count': 3},\n", - " 'Iphone_Os': {'month': 'December', 'count': 4},\n", - " 'Ipados': {'month': 'December', 'count': 4},\n", - " 'Watchos': {'month': 'December', 'count': 1},\n", - " 'Tvos': {'month': 'October', 'count': 1}},\n", - " 'Juniper': {'Junos': {'month': 'January', 'count': 90}},\n", - " 'Liferay': {},\n", - " 'Ivanti': {'Connect_Secure': {'month': 'January', 'count': 44},\n", - " 'Policy_Secure': {'month': 'January', 'count': 37},\n", - " 'Endpoint_Manager': {'month': 'September', 'count': 16},\n", - " 'Avalanche': {'month': 'August', 'count': 20}}}}}" - ] - }, - "execution_count": 123, - "metadata": {}, - "output_type": "execute_result" + "ename": "NameError", + "evalue": "name 'top_month' is not defined", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[5], line 85\u001B[0m\n\u001B[1;32m 78\u001B[0m top_month \u001B[38;5;241m=\u001B[39m (\n\u001B[1;32m 79\u001B[0m product_critical\u001B[38;5;241m.\u001B[39mgroupby(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mPublished_Month\u001B[39m\u001B[38;5;124m'\u001B[39m)\u001B[38;5;241m.\u001B[39msize()\u001B[38;5;241m.\u001B[39midxmax()\n\u001B[1;32m 80\u001B[0m )\n\u001B[1;32m 81\u001B[0m count \u001B[38;5;241m=\u001B[39m (\n\u001B[1;32m 82\u001B[0m product_critical\u001B[38;5;241m.\u001B[39mgroupby(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mPublished_Month\u001B[39m\u001B[38;5;124m'\u001B[39m)\u001B[38;5;241m.\u001B[39msize()\u001B[38;5;241m.\u001B[39mmax()\n\u001B[1;32m 83\u001B[0m )\n\u001B[1;32m 84\u001B[0m critical_spikes[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m2024\u001B[39m\u001B[38;5;124m\"\u001B[39m][vendor][product_name] \u001B[38;5;241m=\u001B[39m {\n\u001B[0;32m---> 85\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmonth\u001B[39m\u001B[38;5;124m\"\u001B[39m: pd\u001B[38;5;241m.\u001B[39mto_datetime(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m2024-\u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[43mtop_month\u001B[49m\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m-01\u001B[39m\u001B[38;5;124m\"\u001B[39m)\u001B[38;5;241m.\u001B[39mstrftime(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m%\u001B[39m\u001B[38;5;124mB\u001B[39m\u001B[38;5;124m'\u001B[39m),\n\u001B[1;32m 86\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcount\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;28mint\u001B[39m(count)\n\u001B[1;32m 87\u001B[0m }\n\u001B[1;32m 89\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m vendor \u001B[38;5;129;01min\u001B[39;00m top_vendors_2023[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mVendor\u001B[39m\u001B[38;5;124m'\u001B[39m]:\n\u001B[1;32m 90\u001B[0m critical_spikes[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m2023\u001B[39m\u001B[38;5;124m\"\u001B[39m][vendor] \u001B[38;5;241m=\u001B[39m {}\n", + "\u001B[0;31mNameError\u001B[0m: name 'top_month' is not defined" + ] } ], - "execution_count": 123 + "execution_count": 5 }, { "metadata": {}, "cell_type": "markdown", "source": [ - "# CISA KEV Analysis\n", + "## CISA KEV Analysis\n", "\n", - "This section evaluates the relationship between NVD data and the CISA KEV catalog:\n", + "The Known Exploited Vulnerabilities (KEV) catalog provides critical insights for organizations to prioritize patching based on active exploitation. This analysis includes:\n", "\n", - "1. **Vulnerabilities Added to CISA KEV**: Counts vulnerabilities added to the KEV catalog, with YoY comparison.\n", - "2. **CISA KEV Overlap with NVD**: Shows percentage of NVD vulnerabilities that are also in the KEV catalog.\n", - "3. **Top Exploited Vendors in CISA KEV**: Ranks vendors with the most KEV vulnerabilities.\n", - "4. **Time to CISA KEV Inclusion**: Calculates time between NVD publication and KEV inclusion.\n" + "1. **Vulnerabilities Added to CISA KEV**: Tracks how many vulnerabilities were added to KEV for 2023 and 2024.\n", + "2. **CISA KEV Overlap with NVD**: Calculates the percentage of NVD vulnerabilities also present in the KEV catalog.\n", + "3. **Top Exploited Vendors in CISA KEV**: Identifies vendors most impacted by KEV vulnerabilities.\n", + "4. **Time to CISA KEV Inclusion**: Measures how quickly vulnerabilities transition from publication to KEV inclusion.\n" ], "id": "949435cad65ba2af" }, - { - "metadata": {}, - "cell_type": "markdown", - "source": [ - "# Specific CVE Details\n", - "\n", - "This section highlights vulnerabilities with high impact or severity:\n", - "\n", - "1. **Most Severe Vulnerabilities**: Lists CVEs with the highest CVSS scores.\n", - "2. **Most Impactful Vulnerabilities**: Combines multiple factors (CVSS, KEV inclusion, exploitation evidence) to rank vulnerabilities.\n" - ], - "id": "cb0b0295d2e5977e" - }, { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:24.053419Z", + "end_time": "2024-12-30T11:23:12.028516Z", "start_time": "2024-12-29T12:09:23.994273Z" } }, @@ -994,7 +864,7 @@ "# Ensure datetime columns are in the correct format and remove timezone info\n", "for df in [kev_records_2023, kev_records_2024]:\n", " df['KEV_DateAdded'] = pd.to_datetime(df['KEV_DateAdded'], errors='coerce').dt.tz_localize(None)\n", - " df['Published_Date'] = pd.to_datetime(df['Published_Date'], errors='coerce').dt.tz_localize(None)\n", + "df['Published_Date'] = pd.to_datetime(df['Published_Date'], errors='coerce').dt.tz_localize(None)\n", "\n", "# Group CISA KEV Data by Month for 2023 and 2024\n", "kev_additions = {\n", @@ -1050,12 +920,12 @@ "vendor_rank_changes = []\n", "for vendor in {v['KEV_Vendor'] for v in top_kev_vendors['2024']}:\n", " rank_2023 = next((i + 1 for i, v in enumerate(top_kev_vendors['2023']) if v['KEV_Vendor'] == vendor), None)\n", - " rank_2024 = next((i + 1 for i, v in enumerate(top_kev_vendors['2024']) if v['KEV_Vendor'] == vendor), None)\n", - " vendor_rank_changes.append({\n", - " \"vendor\": vendor,\n", - " \"2023_rank\": rank_2023,\n", - " \"2024_rank\": rank_2024\n", - " })\n", + "rank_2024 = next((i + 1 for i, v in enumerate(top_kev_vendors['2024']) if v['KEV_Vendor'] == vendor), None)\n", + "vendor_rank_changes.append({\n", + " \"vendor\": vendor,\n", + " \"2023_rank\": rank_2023,\n", + " \"2024_rank\": rank_2024\n", + "})\n", "\n", "# Sort the vendor_rank_changes by 2024 rank\n", "vendor_rank_changes = sorted(vendor_rank_changes, key=lambda x: x[\"2024_rank\"] if x[\"2024_rank\"] else float('inf'))\n", @@ -1064,31 +934,43 @@ "time_to_kev_inclusion = {}\n", "for year, records in {\"2023\": kev_records_2023, \"2024\": kev_records_2024}.items():\n", " inclusion_times = records[records['KEV_DateAdded'].dt.year == int(year)].copy()\n", - " inclusion_times['Time_To_KEV'] = (\n", - " inclusion_times['KEV_DateAdded'] - inclusion_times['Published_Date']\n", - " ).dt.days\n", - " inclusion_times = inclusion_times[inclusion_times['Time_To_KEV'] >= 0]\n", - " time_to_kev_inclusion[year] = {\n", - " \"min_days\": int(inclusion_times['Time_To_KEV'].min()) if not inclusion_times.empty else None,\n", - " \"max_days\": int(inclusion_times['Time_To_KEV'].max()) if not inclusion_times.empty else None,\n", - " \"average_days\": round(float(inclusion_times['Time_To_KEV'].mean()), 2) if not inclusion_times.empty else None\n", - " }\n", + "inclusion_times['Time_To_KEV'] = (\n", + " inclusion_times['KEV_DateAdded'] - inclusion_times['Published_Date']\n", + ").dt.days\n", + "inclusion_times = inclusion_times[inclusion_times['Time_To_KEV'] >= 0]\n", + "time_to_kev_inclusion[year] = {\n", + " \"min_days\": int(inclusion_times['Time_To_KEV'].min()) if not inclusion_times.empty else None,\n", + " \"max_days\": int(inclusion_times['Time_To_KEV'].max()) if not inclusion_times.empty else None,\n", + " \"average_days\": round(float(inclusion_times['Time_To_KEV'].mean()), 2) if not inclusion_times.empty else None\n", + "}\n", "\n", "# Final JSON Structure\n", "cisa_kev = {\n", - " \"kev_additions\": kev_additions,\n", - " \"kev_monthly_changes\": kev_monthly_changes,\n", - " \"nvd_kev_overlap\": {\n", - " \"data\": kev_overlap,\n", - " \"note\": \"Percentage of NVD vulnerabilities in a month that were also added to KEV.\"\n", + " \"metadata\": {\n", + " \"description\": \"Analysis of CISA KEV catalog inclusion and overlap with NVD vulnerabilities for 2023 and 2024.\",\n", + " \"generated_on\": generated_date,\n", + " \"source\": [\"NVD\", \"CISA KEV\"],\n", + " \"attribution\": {\n", + " \"NVD\": \"This product uses the NVD API but is not endorsed or certified by the NVD.\",\n", + " \"CISA KEV\": \"Data from CISA KEV is used under the Creative Commons 0 1.0 License.\"\n", + " },\n", + " \"author\": \"2024 Vulnerability Insights Project\"\n", " },\n", - " \"top_kev_vendors\": top_kev_vendors,\n", - " \"vendor_rank_changes\": vendor_rank_changes,\n", - " \"time_to_kev_inclusion\": time_to_kev_inclusion\n", + " \"data\": {\n", + " \"kev_additions\": kev_additions,\n", + " \"kev_monthly_changes\": kev_monthly_changes,\n", + " \"nvd_kev_overlap\": {\n", + " \"data\": kev_overlap,\n", + " \"note\": \"Percentage of NVD vulnerabilities in a month that were also added to KEV.\"\n", + " },\n", + " \"top_kev_vendors\": top_kev_vendors,\n", + " \"vendor_rank_changes\": vendor_rank_changes,\n", + " \"time_to_kev_inclusion\": time_to_kev_inclusion\n", + " }\n", "}\n", "\n", "# Save the cisa_kev metrics to a JSON file\n", - "with open(\"../../data/2024_insights/output/cisa_kev.json\", \"w\") as f:\n", + "with open(\"../../data/2024_insights/output/cisa_kev_analysis.json\", \"w\") as f:\n", " json.dump(cisa_kev, f)\n", "\n", "cisa_kev" @@ -1218,9 +1100,9 @@ "metadata": {}, "cell_type": "markdown", "source": [ - "# Specific CVE Details\n", + "## Specific CVE Details\n", "\n", - "This section highlights vulnerabilities with high impact or severity:\n", + "This section highlights vulnerabilities with high impact or severity to assist in prioritization and understanding of high-risk CVEs:\n", "\n", "1. **Most Severe Vulnerabilities**: Lists CVEs with the highest CVSS scores.\n", "2. **Most Impactful Vulnerabilities**: Combines multiple factors (CVSS, KEV inclusion, exploitation evidence) to rank vulnerabilities." @@ -1230,7 +1112,7 @@ { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:25.212520Z", + "end_time": "2024-12-30T11:23:12.053024Z", "start_time": "2024-12-29T12:09:24.305873Z" } }, @@ -1249,9 +1131,9 @@ " .drop_duplicates()\n", " .head(remaining_cves_needed)\n", " )\n", - " most_severe = pd.concat([cvss_10_cves, additional_cves]).drop_duplicates()\n", + "most_severe = pd.concat([cvss_10_cves, additional_cves]).drop_duplicates()\n", "else:\n", - " most_severe = cvss_10_cves\n", + "most_severe = cvss_10_cves\n", "\n", "# Ensure the final result is sorted and unique\n", "most_severe = (\n", @@ -1266,6 +1148,8 @@ "\n", "\n", "# Define Impact Score calculation function\n", + "\n", + "\n", "def calculate_impact_score(row):\n", " exploitation_weight = 10 if row['Exploitation_Evidence'] else 0\n", " impact_score = row['CVSS_Base_Score'] * 2 + exploitation_weight\n", @@ -1286,16 +1170,28 @@ "\n", "# Final JSON structure\n", "specific_cve_details = {\n", - " \"most_severe\": most_severe,\n", - " \"most_impactful\": most_impactful,\n", - " \"notes\": {\n", - " \"most_severe\": \"Includes all CVEs with CVSS_Base_Score of 10.0 and additional CVEs to make a total of 25, if fewer than 25 CVSS 10.0 CVEs exist.\",\n", - " \"most_impactful\": \"Top 10 CVEs by impact score, prioritizing exploitation evidence.\"\n", + " \"metadata\": {\n", + " \"description\": \"Detailed analysis of the most severe and impactful CVEs for 2024.\",\n", + " \"generated_on\": generated_date,\n", + " \"source\": [\"NVD\", \"CISA KEV\"],\n", + " \"attribution\": {\n", + " \"NVD\": \"This product uses the NVD API but is not endorsed or certified by the NVD.\",\n", + " \"CISA KEV\": \"Data from CISA KEV is used under the Creative Commons 0 1.0 License.\"\n", + " },\n", + " \"author\": \"2024 Vulnerability Insights Project\"\n", + " },\n", + " \"data\": {\n", + " \"most_severe\": most_severe,\n", + " \"most_impactful\": most_impactful,\n", + " \"notes\": {\n", + " \"most_severe\": \"Includes all CVEs with CVSS_Base_Score of 10.0 and additional CVEs to make a total of 25, if fewer than 25 CVSS 10.0 CVEs exist.\",\n", + " \"most_impactful\": \"Top 10 CVEs by impact score, prioritizing exploitation evidence.\"\n", + " }\n", " }\n", "}\n", "\n", "# Save the specific_cve_details metrics to a JSON file\n", - "with open(\"../../data/2024_insights/output/specific_cve_details.json\", \"w\") as f:\n", + "with open(\"../../data/2024_insights/output/cve_details.json\", \"w\") as f:\n", " json.dump(specific_cve_details, f)\n", "\n", "specific_cve_details" @@ -1480,18 +1376,19 @@ "metadata": {}, "cell_type": "markdown", "source": [ - "# CVE Assigner Analysis\n", + "## CVE Assigner Analysis\n", "\n", - "This section identifies the organizations assigning the most CVEs:\n", + "This section identifies the organizations assigning the most CVEs to understand trends and priorities in vulnerability reporting.\n", "\n", - "1. **Top CVE Assigners**: Highlights top assigners by the number of CVEs and severity breakdown (Critical and High).\n" + "1. **Top CVE Assigners**: Highlights top assigners by the number of CVEs and severity breakdown (Critical, High, Medium, Low).\n", + "2. **Year-over-Year Comparison**: Provides insights into changes in CVE assignment trends between 2023 and 2024." ], "id": "4076df948af5b296" }, { "metadata": { "ExecuteTime": { - "end_time": "2024-12-29T12:09:25.670248Z", + "end_time": "2024-12-30T11:23:12.054709Z", "start_time": "2024-12-29T12:09:25.221578Z" } }, @@ -1543,19 +1440,31 @@ "\n", "# Transform to JSON-friendly structure\n", "top_assigners = {\n", - " \"top_assigners\": {\n", - " \"2023\": top_assigners_2023.to_dict(orient='records'),\n", - " \"2024\": top_assigners_2024.to_dict(orient='records'),\n", - " \"comparison_notes\": (\n", - " \"Critical: CVSS >= 9.0, High: 7.0 <= CVSS < 9.0, \"\n", - " \"Medium: 4.0 <= CVSS < 7.0, Low: CVSS < 4.0. \"\n", - " \"Shows top assigners for each year with severity breakdowns, \"\n", - " \"total counts, and year-over-year changes.\"\n", - " ),\n", - " \"totals\": {\n", - " \"2023\": total_2023,\n", - " \"2024\": total_2024,\n", - " \"percentage_changes\": percentage_changes\n", + " \"metadata\": {\n", + " \"description\": \"Analysis of top CVE assigners and severity breakdowns for 2023 and 2024.\",\n", + " \"generated_on\": generated_date,\n", + " \"source\": [\"NVD\", \"CISA KEV\"],\n", + " \"attribution\": {\n", + " \"NVD\": \"This product uses the NVD API but is not endorsed or certified by the NVD.\",\n", + " \"CISA KEV\": \"Data from CISA KEV is used under the Creative Commons 0 1.0 License.\"\n", + " },\n", + " \"author\": \"2024 Vulnerability Insights Project\"\n", + " },\n", + " \"data\": {\n", + " \"top_assigners\": {\n", + " \"2023\": top_assigners_2023.to_dict(orient='records'),\n", + " \"2024\": top_assigners_2024.to_dict(orient='records'),\n", + " \"comparison_notes\": (\n", + " \"Critical: CVSS >= 9.0, High: 7.0 <= CVSS < 9.0, \"\n", + " \"Medium: 4.0 <= CVSS < 7.0, Low: CVSS < 4.0. \"\n", + " \"Shows top assigners for each year with severity breakdowns, \"\n", + " \"total counts, and year-over-year changes.\"\n", + " ),\n", + " \"totals\": {\n", + " \"2023\": total_2023,\n", + " \"2024\": total_2024,\n", + " \"percentage_changes\": percentage_changes\n", + " }\n", " }\n", " }\n", "}\n",