diff --git a/.gitignore b/.gitignore index 68bc17f..64791c5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ __pycache__/ # C extensions *.so +#vscode +.vscode +.vscode/ +vscode +vscode/ # Distribution / packaging .Python build/ @@ -158,3 +163,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +.vscode/settings.json diff --git a/GMeasurements/__init__.py b/GMeasurements/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/GMeasurements/measurement_params.py b/GMeasurements/measurement_params.py new file mode 100644 index 0000000..aa105da --- /dev/null +++ b/GMeasurements/measurement_params.py @@ -0,0 +1,152 @@ +from typing import Any, Dict, List, Optional, Unpack, Required, Optional, TypedDict, Union, NotRequired,overload +from datetime import datetime + +class GetMeasurementParams(TypedDict): + sort: NotRequired[str] + id__lt: NotRequired[str] + id__lte: NotRequired[str] + id__gt: NotRequired[str] + id__gte: NotRequired[str] + id__in: NotRequired[str] + id: NotRequired[str] + start_time: NotRequired[datetime] + start_time__lt: NotRequired[datetime] + start_time__lte: NotRequired[datetime] + start_time__gt: NotRequired[datetime] + start_time__gte: NotRequired[datetime] + stop_time: NotRequired[datetime] + stop_time__lt: NotRequired[datetime] + stop_time__lte: NotRequired[datetime] + stop_time__gt: NotRequired[datetime] + stop_time__gte: NotRequired[datetime] + is_public: NotRequired[bool] + is_oneoff: NotRequired[bool] + interval: NotRequired[str] + interval__lt: NotRequired[str] + interval__lte: NotRequired[str] + interval__gt: NotRequired[str] + interval__gte: NotRequired[str] + status: NotRequired[str] + status__in: NotRequired[str] + tags: NotRequired[str] + type: NotRequired[str] + target_ip: NotRequired[str] + current_probes: NotRequired[str] + participant_logs_probes: NotRequired[str] + target_asn: NotRequired[str] + target: NotRequired[str] + target__contains: NotRequired[str] + target__startswith: NotRequired[str] + target__endswith: NotRequired[str] + description: NotRequired[str] + description__contains: NotRequired[str] + description__startswith: NotRequired[str] + description__endswith: NotRequired[str] + af: NotRequired[str] + search: NotRequired[str] + protocol: NotRequired[str] + group_id: NotRequired[str] + group: NotRequired[str] + favourite: NotRequired[bool] + hidden: NotRequired[bool] + page: NotRequired[int] + page_size: NotRequired[int] + after: NotRequired[str] + format: NotRequired[str] + callback: NotRequired[str] + optional_fields: NotRequired[str] + fields: NotRequired[str] + format_datetime: NotRequired[str] + key: NotRequired[str] + mine: NotRequired[bool] + +class MeasurementParams(TypedDict): + description: Required[str] + af: Required[int] + type: Required[str] + resolve_on_probe: NotRequired[bool] + is_oneoff: NotRequired[bool] + start_time: NotRequired[datetime] + stop_time: NotRequired[datetime] + interval: NotRequired[int] + spread: NotRequired[int] + is_public: NotRequired[bool] + +class TracerouteParams(MeasurementParams): + target: Required[str] + response_timeout: NotRequired[int] + packets: NotRequired[int] + paris: NotRequired[int] + size: NotRequired[int] + first_hop: NotRequired[int] + max_hops: NotRequired[int] + protocol: NotRequired[str] + +class SSLCertParams(MeasurementParams): + target: Required[str] + +class HTTPParams(MeasurementParams): + target: Required[str] + +class NTPParams(MeasurementParams): + target: Required[str] + +class PingParams(MeasurementParams): + target: Required[str] + packets: NotRequired[int] + size: NotRequired[int] + packet_interval: NotRequired[int] + include_probe_id: NotRequired[bool] + +class DNSParams(MeasurementParams): + query_class: Required[str] + query_type: Required[str] + query_argument: Required[str] + target_server: NotRequired[str] + timeout: NotRequired[int] + udp_payload_size: NotRequired[int] + retry_times: NotRequired[int] + protocol: NotRequired[str] + use_probes_resolver: NotRequired[bool] + +class GetProbesParams(TypedDict): + country_code: NotRequired[str] + id__lt: NotRequired[int] + id__lte: NotRequired[int] + id__gte: NotRequired[int] + id__gt: NotRequired[int] + id__in: NotRequired[str] + latitude: NotRequired[str] + latitude__lt: NotRequired[str] + latitude__lte: NotRequired[str] + latitude__gte: NotRequired[str] + latitude__gt: NotRequired[str] + longitude: NotRequired[str] + longitude__lt: NotRequired[str] + longitude__lte: NotRequired[str] + longitude__gte: NotRequired[str] + longitude__gt: NotRequired[str] + asn: NotRequired[str] + asn_v4: NotRequired[str] + asn_v4__in: NotRequired[str] + asn_v6: NotRequired[str] + asn_v6__in: NotRequired[str] + prefix_v4: NotRequired[str] + prefix_v6: NotRequired[str] + status: NotRequired[str] + status_name: NotRequired[str] + is_anchor: NotRequired[bool] + is_public: NotRequired[bool] + tags: NotRequired[str] + radius: NotRequired[str] + country_code__in: NotRequired[str] + include: NotRequired[str] + optional_fields: NotRequired[str] + format: NotRequired[str] + sort: NotRequired[str] + +class ProbeParams(TypedDict): + requested: Required[int] + type: Required[str] + value: Required[Union[str,int]] + diff --git a/GMeasurements/measurements.py b/GMeasurements/measurements.py new file mode 100644 index 0000000..91d9773 --- /dev/null +++ b/GMeasurements/measurements.py @@ -0,0 +1,266 @@ +from datetime import datetime +import pandas as pd +""" + + +For more information on the ripe atlas rest api +https://atlas.ripe.net/docs/apis/rest-api-manual/ + + +""" +from typing import Any, Dict, List, Unpack, Required, NotRequired +from requests import request, get, post +import json +from measurement_params import * +from enum import Enum, auto +from dotenv import load_dotenv +from os import getenv +load_dotenv() + +class Payload(): + def __init__(self): + self.definitions: List[Dict[str, Any]] = [] + self.probes: List[Dict[str, Any]] = [] + self.payload: Dict[str, Any] = {} + self.base_measurement_keys ={ + "description", "af", "type", "resolve_on_probe", "is_oneoff", "start_time", "stop_time", "interval", "spread", "is_public" + } + self.base_required_keys = { + "description", "af", "type" + } + self.base_start_time = None + self.base_end_time = None + self.base_is_one_off = None + + def _add_definition(self, required_keys, valid_type, all_keys, **kwargs): + if not required_keys.issubset(kwargs): + raise ValueError(f"Missing required keys: {required_keys - kwargs.keys()}") + + if kwargs["type"] != valid_type: + raise ValueError(f"Type must be '{valid_type}'") + + definition = {key: kwargs[key] for key in all_keys if key in kwargs} + self.definitions.append(definition) + + def add_traceroute_definition(self, **kwargs: Unpack[TracerouteParams]) -> None: + traceroute_required_keys = {"target"} + traceroute_all_keys = { + "target", "response_timeout", "packets", "paris", "size", "first_hop", "max_hops", "protocol" + } + required_keys = traceroute_required_keys | self.base_required_keys + all_keys = traceroute_all_keys | self.base_measurement_keys + self._add_definition(required_keys, "traceroute", all_keys, **kwargs) + + def add_ping_definition(self, **kwargs: Unpack[PingParams]) -> None: + ping_required_keys = {"target"} + ping_all_keys = { + "target", "packets", "size", "packet_interval", "include_probe_id" + } + required_keys = ping_required_keys | self.base_required_keys + all_keys = ping_all_keys | self.base_measurement_keys + self._add_definition(required_keys, "ping", all_keys, **kwargs) + + def add_probe(self, **kwargs:Unpack[PingParams]) -> None: + """ + Adds a probe to the payload. + + Args: + **kwargs (PingParams): The probe parameters. + + """ + probe = {} + required_keys = {"requested", "type", "value"} + if not required_keys.issubset(kwargs): + raise ValueError(f"Missing required keys: {required_keys - kwargs.keys()}") + types = {"probes", "area", "country", "msm", "prefix", "asn"} + if kwargs["type"] not in types: + raise ValueError(f"Type must be one of: {types}") + probe = {key: kwargs[key] for key in required_keys if key in kwargs} + self.probes.append(probe) + + def get_payload(self) -> Dict[str, Any]: + """ + Returns the payload in the format required by the RIPE Atlas API. + + Returns: + Dict[str, Any]: The payload in the format required by the RIPE Atlas API. + """ + if len(self.definitions) == 0: + raise ValueError("No definitions found") + if len(self.probes) == 0: + raise ValueError("No probes found") + return { + "definitions": self.definitions, + "probes": self.probes + } + + +class RipeAtlasMeasurements(): + def __init__(self, ATLAS_API_KEY): + self.url = "https://atlas.ripe.net/api/v2/" + self.headers = { + "Content-Type": "application/json", + "Authorization" : f"Key {ATLAS_API_KEY}" + } + self.key = ATLAS_API_KEY + + def _post(self, base_url:str, payload:dict): + """ + Performs a post request to the RIPE Atlas v2 API. + + Args: + base_url (string): _description_ + type (_type_, optional): _description_. Defaults to None. + + Returns: + list: The response from the API if successful or an error message if unsuccessful. + """ + + request_url = self.url + base_url + response = post(url=request_url, headers=self.headers, data=json.dumps(payload)) + + if(response.status_code == 201): + return response.json() + else: + return {"error": response.json()} + + def _get_json_response(self, base_url, type=None): + """ + Performs a GET request to the RIPE Atlas v2 API. + + Args: + base_url (string): The portion of the url that defines what information is retrieved from the API. + type (_type_, optional): _description_. Defaults to None. + + Returns: + list/dict: If successful, returns the response from the API, which can be a list or a dict. + dict: If GET request unsuccessful, returns a dict with the error message. + """ + request_url = self.url + base_url + response = get(request_url) + if(response.status_code == 200): + return response.json() + else: + return {"error": f"Returned with status code {response.status_code}"} + + def get_generic_measurement(self, msm_id:str): + base_url = f"measurements/{msm_id}/" + response = self._get_json_response(base_url=base_url) + return response + + def get_measurement_result(self, msm_id:str, start:datetime=None, stop:datetime=None): + """_summary_ + + Args: + msm_id (str): _description_ + start (datetime.datetime, optional): _description_. Defaults to None. + stop (datetime.datetime, optional): _description_. Defaults to None. + + Returns: + dict: The results of the measurement in json format. + """ + + base_url = f"measurements/{msm_id}/results/" + + if start and stop: + base_url += f"?start_time={start.timestamp()}&stop_time={stop.timestamp()}" + elif start: + base_url += f"?start_time={start.timestamp()}" + elif stop: + base_url += f"?stop_time={stop.timestamp()}" + + response = self._get_json_response(base_url=base_url) + if type(response) == dict: + raise ValueError(f"Bad Request in get_probes with error: {response['error']}") + + return response # CAUTION: we are only concerned with one-off traceroute measurements, hence we only grab the first result in the list, which is the only result with one-off traceroute measurements + + def create_measurement(self, type, payload:Payload): + base_url = "measurements/" + measurement= self._post(base_url=base_url, payload=payload.get_payload()) # response is a dict with one field: measurements: [list of measurement ids] + if "error" in measurement: + raise ValueError(f"Bad Request in create_measurement with error: {measurement['error']}") + return measurement["measurements"] + + def get_probes(self, **kwargs:Unpack[GetProbesParams]) -> list: + """ + Returns a list of all probes based on the given parameters. + + Args: + country_code, id__in, asn, asn_v4, asn_v4__in, asn_v6, asn_v6__in, + prefix_v4, prefix_v6, status, status_name, tags, include, + optional_fields, format, sort (str, Optional): + Filters probes based on the specified string values. + + id__lt, id__lte, id__gte, id__gt (int, Optional): + Filters probes based on ID values, e.g., id__lt filters for probes with IDs less than the specified value. + + latitude, latitude__lt, latitude__lte, latitude__gte, latitude__gt (str, Optional): + Filters probes based on latitude values. `__lt` and `__lte` filter for probes south of the specified value, + while `__gte` and `__gt` filter for probes north of the specified value. + + longitude, longitude__lt, longitude__lte, longitude__gte, longitude__gt (str, Optional): + Filters probes based on longitude values. `__lt` and `__lte` filter for probes west of the specified value, + while `__gte` and `__gt` filter for probes east of the specified value. + + is_anchor, is_public (bool, Optional): + Filters probes based on their anchor or public status. + + radius (str, Optional): + Filters probes within the specified radius of the supplied latitude and longitude. + + status (int, Optional): + Probe status (0-Never Connected, 1-Connected, 2-Disconnected, 3-Abandoned). + + + Raises: + ValueError: Raised if the request to the Ripe Atlas API is unsuccessful. + + Returns: + List[Dict[str, Any]]: A list of probes matching the given parameters. Each probe is represented as: + {\n + "id": probe_id,\n + "ipv4": ipv4_address,\n + "country_code": country_code,\n + "geometry": {"type": type, "coordinates": [lat, lon]},\n + "status": {"id": connected_code, "name": connected_status_name, "since": date}\n + } + """ + + base_url = "probes/" + query_params="" + IS_PARAMS = False + for (key,value) in kwargs.items(): + if not IS_PARAMS: + query_params += "?" + IS_PARAMS = True + + base_url += query_params + data = self._get_json_response(base_url) + if "error" in data.keys(): + raise ValueError(f"Bad Request in get_probes with error: {data['error']}") + total_probes_returned = data["count"] + probes_list = [] + for probe in data["results"]: + probe_data = {} + probe_data["id"] = probe["id"] + probe_data["ipv4"] = probe["address_v4"] + probe_data["country_code"] = probe["country_code"] + probe_data["geometry"] = probe["geometry"] + probe_data["status"] = probe["status"] + probe_data["type"] = probe['type'] + probes_list.append(probe_data) + return probes_list + + +create_measurement = RipeAtlasMeasurements("4001f1c2-f49d-4727-85f6-62322b76eaac") + + +measurement = create_measurement.get_measurement_result("62390085") +measurement2 = create_measurement.get_generic_measurement("62390085") + + +mdf = pd.DataFrame(measurement) + +print(mdf.head()) +mdf.to_csv("test.csv",mode='w',index=False) \ No newline at end of file diff --git a/GMeasurements/ripe_atlas_parser.py b/GMeasurements/ripe_atlas_parser.py new file mode 100644 index 0000000..5773318 --- /dev/null +++ b/GMeasurements/ripe_atlas_parser.py @@ -0,0 +1,8 @@ +from logger import Logger + + +def parse_traceroute_measurement(): + pass + +def parse_ping_measurement(): + pass \ No newline at end of file diff --git a/README.md b/README.md index 70beee3..e6dd071 100644 --- a/README.md +++ b/README.md @@ -1 +1,4 @@ -# CSCEA365-Group-Projects \ No newline at end of file +# CSCEA365-Group-Projects + +run the following to install all requirements +pip install -r requirements.txt \ No newline at end of file diff --git a/data/alaska_probes/alaska_probes.txt b/data/alaska_probes/alaska_probes.txt new file mode 100644 index 0000000..1fec84b --- /dev/null +++ b/data/alaska_probes/alaska_probes.txt @@ -0,0 +1,17 @@ +i +Filters: + radius: 61.217,-149.863:1300 + status: 1 + + id asn_v4 asn_v6 country status +=========================================== + 1118 6058 - ca Connected +14300 8047 - us Connected +51310 8047 - us Connected +51754 8047 - us Connected +52344 11090 - us Connected +60259 8047 8047 us Connected +61113 14593 - us Connected +61868 20057 - us Connected +=========================================== + Showing 8 of 8 diff --git a/data/alaska_probes/remote_alaska_public_ips.csv b/data/alaska_probes/remote_alaska_public_ips.csv new file mode 100644 index 0000000..058acf4 --- /dev/null +++ b/data/alaska_probes/remote_alaska_public_ips.csv @@ -0,0 +1,13 @@ +description,ip,date verified,accuracy of being remote AK,comments +Kotzebue,216.163.106.1,2023-11-6,yes +Kotzebue,74.127.92.1,2023-11-6,yes +Kotzebue,74.127.92.2,2023-11-6,yes +Nome,67.59.96.3,2023-11-6,yes +Bethel,24.237.58.1,2023-11-6,yes but dynamic +Alakanuk,67.58.30.2,2023-11-6,not sure +Metlakatla,64.186.125.1,2023-11-6,yes +Dillingham,107.152.126.47,2023-11-6,not sure +Chevak,24.237.232.2,2023-11-6,yes,resolves to chevak.kasd.schoolaccess.net + + + diff --git a/data/measurements/measurements.csv b/data/measurements/measurements.csv new file mode 100644 index 0000000..4dcd287 --- /dev/null +++ b/data/measurements/measurements.csv @@ -0,0 +1,26 @@ +source,target,measurement_id +"14300,52344,61868,61113","165.16.221.228",61984614 +"14300,52344,61868,61113","181.215.183.179",61984615 +"14300,52344,61868,61113","49.36.191.11",61984617 +"14300,52344,61868,61113","116.202.28.156",61984618 +"14300,52344,61868,61113","103.21.3.89",61984619 +"14300,51310,51754,52344,60259,61113,61868",165.16.221.228,62028101 +"14300,51310,51754,52344,60259,61113,61868",181.215.183.179,62028103 +"14300,51310,51754,52344,60259,61113,61868",49.36.191.11,62028104 +"14300,51310,51754,52344,60259,61113,61868",116.202.28.156,62028105 +"14300,51310,51754,52344,60259,61113,61868",103.21.3.89,62028107 +"14300,51310,51754,52344,60259,61113,61868",165.16.221.228,62028891 +"14300,51310,51754,52344,60259,61113,61868",181.215.183.179,62028893 +"14300,51310,51754,52344,60259,61113,61868",49.36.191.11,62028894 +"14300,51310,51754,52344,60259,61113,61868",116.202.28.156,62028896 +"14300,51310,51754,52344,60259,61113,61868",103.21.3.89,62028897 +"14300,51310,51754,52344,60259,61113,61868",165.16.221.228,62029175 +"14300,51310,51754,52344,60259,61113,61868",181.215.183.179,62029177 +"14300,51310,51754,52344,60259,61113,61868",49.36.191.11,62029179 +"14300,51310,51754,52344,60259,61113,61868",116.202.28.156,62029180 +"14300,51310,51754,52344,60259,61113,61868",103.21.3.89,62029181 +"14300,51310,51754,52344,60259,61113,61868",165.16.221.228,62029595 +"14300,51310,51754,52344,60259,61113,61868",181.215.183.179,62029597 +"14300,51310,51754,52344,60259,61113,61868",49.36.191.11,62029600 +"14300,51310,51754,52344,60259,61113,61868",116.202.28.156,62029601 +"14300,51310,51754,52344,60259,61113,61868",103.21.3.89,62029609 diff --git a/data/measurements/test_measurements.csv b/data/measurements/test_measurements.csv new file mode 100644 index 0000000..5322c7e --- /dev/null +++ b/data/measurements/test_measurements.csv @@ -0,0 +1,2 @@ +60259 +63195895 diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..f40fbd8 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,5 @@ +_site +.sass-cache +.jekyll-cache +.jekyll-metadata +vendor diff --git a/docs/404.html b/docs/404.html new file mode 100644 index 0000000..086a5c9 --- /dev/null +++ b/docs/404.html @@ -0,0 +1,25 @@ +--- +permalink: /404.html +layout: default +--- + + + +
+

404

+ +

Page not found :(

+

The requested page could not be found.

+
diff --git a/docs/Gemfile b/docs/Gemfile new file mode 100644 index 0000000..c1bfeea --- /dev/null +++ b/docs/Gemfile @@ -0,0 +1,35 @@ +source "https://rubygems.org" +# Hello! This is where you manage which Jekyll version is used to run. +# When you want to use a different version, change it below, save the +# file and run `bundle install`. Run Jekyll with `bundle exec`, like so: +# +# bundle exec jekyll serve +# +# This will help ensure the proper Jekyll version is running. +# Happy Jekylling! +# gem "jekyll", "~> 4.3.2" +# This is the default theme for new Jekyll sites. You may change this to anything you like. +gem "minima", "~> 2.5" +# If you want to use GitHub Pages, remove the "gem "jekyll"" above and +# uncomment the line below. To upgrade, run `bundle update github-pages`. +gem "github-pages", group: :jekyll_plugins +# If you have any plugins, put them here! +group :jekyll_plugins do + gem "jekyll-feed", "~> 0.12" +end + +# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem +# and associated library. +platforms :mingw, :x64_mingw, :mswin, :jruby do + gem "tzinfo", ">= 1", "< 3" + gem "tzinfo-data" +end + +# Performance-booster for watching directories on Windows +gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] + +# Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem +# do not have a Java counterpart. +gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] + +gem "webrick", "~> 1.7" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock new file mode 100644 index 0000000..753b696 --- /dev/null +++ b/docs/Gemfile.lock @@ -0,0 +1,278 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.1.1) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + minitest (>= 5.1) + mutex_m + tzinfo (~> 2.0) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) + base64 (0.1.1) + bigdecimal (3.1.4) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.23.10) + concurrent-ruby (1.2.2) + connection_pool (2.4.1) + dnsruby (1.70.0) + simpleidn (~> 0.2.1) + drb (2.1.1) + ruby2_keywords + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + ethon (0.16.0) + ffi (>= 1.15.0) + eventmachine (1.2.7) + execjs (2.9.1) + faraday (2.7.11) + base64 + faraday-net_http (>= 2.0, < 3.1) + ruby2_keywords (>= 0.0.4) + faraday-net_http (3.0.2) + ffi (1.16.3) + forwardable-extended (2.6.0) + gemoji (3.0.1) + github-pages (228) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.3) + jekyll-avatar (= 0.7.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.4.0) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.15.1) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) + jekyll-mentions (= 1.6.0) + jekyll-optional-front-matter (= 0.3.2) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.3.0) + jekyll-redirect-from (= 0.16.0) + jekyll-relative-links (= 0.6.1) + jekyll-remote-theme (= 0.4.3) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.8.0) + jekyll-sitemap (= 1.4.0) + jekyll-swiss (= 1.0.0) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) + jekyll-titles-from-headings (= 0.5.3) + jemoji (= 0.12.0) + kramdown (= 2.3.2) + kramdown-parser-gfm (= 1.1.0) + liquid (= 4.0.4) + mercenary (~> 0.3) + minima (= 2.5.1) + nokogiri (>= 1.13.6, < 2.0) + rouge (= 3.26.0) + terminal-table (~> 1.4) + github-pages-health-check (1.17.9) + addressable (~> 2.3) + dnsruby (~> 1.60) + octokit (~> 4.0) + public_suffix (>= 3.0, < 5.0) + typhoeus (~> 1.3) + html-pipeline (2.14.3) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.8.0) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + jekyll (3.9.3) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (>= 0.7, < 2) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (>= 1.17, < 3) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-avatar (0.7.0) + jekyll (>= 3.0, < 5.0) + jekyll-coffeescript (1.1.1) + coffee-script (~> 2.2) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.4.0) + commonmarker (~> 0.23.7) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) + rouge (>= 2.0, < 5.0) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.15.1) + jekyll (>= 3.7, < 5.0) + jekyll-gist (1.5.0) + octokit (~> 4.2) + jekyll-github-metadata (2.13.0) + jekyll (>= 3.4, < 5.0) + octokit (~> 4.0, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-mentions (1.6.0) + html-pipeline (~> 2.3) + jekyll (>= 3.7, < 5.0) + jekyll-optional-front-matter (0.3.2) + jekyll (>= 3.0, < 5.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.3.0) + jekyll (>= 3.0, < 5.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-remote-theme (0.4.3) + addressable (~> 2.0) + jekyll (>= 3.5, < 5.0) + jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) + rubyzip (>= 1.3.0, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-swiss (1.0.0) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.6.0) + jekyll (> 3.5, < 5.0) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.3) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.12.0) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (>= 3.0, < 5.0) + kramdown (2.3.2) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.3.6) + minima (2.5.1) + jekyll (>= 3.5, < 5.0) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.20.0) + mutex_m (0.1.2) + nokogiri (1.15.4-x86_64-linux) + racc (~> 1.4) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + racc (1.7.1) + rb-fsevent (0.11.2) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.6) + rouge (3.26.0) + ruby2_keywords (0.0.5) + rubyzip (2.3.2) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + simpleidn (0.2.1) + unf (~> 0.1.4) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + typhoeus (1.4.0) + ethon (>= 0.9.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) + unicode-display_width (1.8.0) + webrick (1.8.1) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + github-pages + http_parser.rb (~> 0.6.0) + jekyll-feed (~> 0.12) + minima (~> 2.5) + tzinfo (>= 1, < 3) + tzinfo-data + wdm (~> 0.1.1) + webrick (~> 1.7) + +BUNDLED WITH + 2.4.21 diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..85026cd --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,51 @@ +# Welcome to Jekyll! +# +# This config file is meant for settings that affect your whole blog, values +# which you are expected to set up once and rarely edit after that. If you find +# yourself editing this file very often, consider using Jekyll's data files +# feature for the data you need to update frequently. +# +# For technical reasons, this file is *NOT* reloaded automatically when you use +# 'bundle exec jekyll serve'. If you change this file, please restart the server process. +# +# If you need help with YAML syntax, here are some quick references for you: +# https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml +# https://learnxinyminutes.com/docs/yaml/ +# +# Site settings +# These are used to personalize your new site. If you look in the HTML files, +# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. +# You can create any custom variable you would like, and they will be accessible +# in the templates via {{ site.myvariable }}. + +title: Documentation for Ripe Atlas +email: jarenramirez2019@gmail.com +description: >- # this means to ignore newlines until "baseurl:" +baseurl: "" # the subpath of your site, e.g. /blog +url: "localhost:4000" # the base hostname & protocol for your site, e.g. http://example.com + + +# Build settings +theme: minima +plugins: + - jekyll-feed + +# Exclude from processing. +# The following items will not be processed, by default. +# Any item listed under the `exclude:` key here will be automatically added to +# the internal "default list". +# +# Excluded items can be processed by explicitly listing the directories or +# their entries' file path in the `include:` list. +# +# exclude: +# - .sass-cache/ +# - .jekyll-cache/ +# - gemfiles/ +# - Gemfile +# - Gemfile.lock +# - node_modules/ +# - vendor/bundle/ +# - vendor/cache/ +# - vendor/gems/ +# - vendor/ruby/ diff --git a/docs/_layouts/base.html b/docs/_layouts/base.html new file mode 100644 index 0000000..6092d27 --- /dev/null +++ b/docs/_layouts/base.html @@ -0,0 +1,42 @@ + + + + + + + Documentation for Ripe Atlas + + + + + + + + + + {{ content }} + + \ No newline at end of file diff --git a/docs/about/index.html b/docs/about/index.html new file mode 100644 index 0000000..88f00ef --- /dev/null +++ b/docs/about/index.html @@ -0,0 +1,3 @@ +--- +layout: base +--- \ No newline at end of file diff --git a/docs/favicon.ico b/docs/favicon.ico new file mode 100644 index 0000000..4decb70 Binary files /dev/null and b/docs/favicon.ico differ diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..9efcf3f --- /dev/null +++ b/docs/index.html @@ -0,0 +1,4 @@ +--- +layout: base +--- + diff --git a/docs/progress/index.html b/docs/progress/index.html new file mode 100644 index 0000000..88f00ef --- /dev/null +++ b/docs/progress/index.html @@ -0,0 +1,3 @@ +--- +layout: base +--- \ No newline at end of file diff --git a/docs/reference/index.html b/docs/reference/index.html new file mode 100644 index 0000000..88f00ef --- /dev/null +++ b/docs/reference/index.html @@ -0,0 +1,3 @@ +--- +layout: base +--- \ No newline at end of file diff --git a/languages.md b/languages.md new file mode 100644 index 0000000..c56f241 --- /dev/null +++ b/languages.md @@ -0,0 +1 @@ +RUST and Spark \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..53267a4 --- /dev/null +++ b/main.py @@ -0,0 +1,28 @@ +from GMeasurements.measurements import RipeAtlasMeasurements +from dotenv import load_dotenv +from os import getenv +import json +import csv + +if __name__ == "__main__": + load_dotenv() + auth_key = getenv("ATLAS_API_KEY","NONE") + if auth_key == "None": + raise("error getting atlas api key") + + # create an instance of class RipeAtlasMeasurement that can be used for a lot of different measurements + measurement = RipeAtlasMeasurements(ATLAS_API_KEY=auth_key) + + # TESTING SUCCESSFUL: get_traceroute_measurement + # msm1 = measurement.get_traceroute_measurement(61984619) + + # TESTING SUCCESSFUL: create_traceroute_measurement + probe_list = [60259] + new_measurement = measurement.create_traceroute_measurement(description="testing", target="ripe.net", af=4, probe_ids=probe_list) + + # appends measurement ids to this csv + with open("data/measurements/test_measurements.csv", mode='a', newline='') as f: + writer = csv.writer(f) + for measurement in new_measurement: + writer.writerow([measurement]) + diff --git a/pandas_helpers/__init__.py b/pandas_helpers/__init__.py new file mode 100644 index 0000000..afa90ae --- /dev/null +++ b/pandas_helpers/__init__.py @@ -0,0 +1 @@ +from .edit import remove_columns \ No newline at end of file diff --git a/pandas_helpers/edit.py b/pandas_helpers/edit.py new file mode 100644 index 0000000..fbdc3e9 --- /dev/null +++ b/pandas_helpers/edit.py @@ -0,0 +1,15 @@ +import pandas as pd + +def remove_columns(dataframe, *args): + for col_name in args: + dataframe.drop(col_name,axis=1,inplace=True) + +def change_column_position(dataframe, col, place:int): + columns = list(dataframe.columns) + try: + if col in columns: + columns.remove(col) + columns.insert(1,col) + dataframe = dataframe[columns] + except: + raise ValueError("Column did not exist") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e258edf Binary files /dev/null and b/requirements.txt differ diff --git a/tests/test.csv b/tests/test.csv new file mode 100644 index 0000000..51a193a --- /dev/null +++ b/tests/test.csv @@ -0,0 +1,11 @@ +fw,mver,lts,endtime,dst_name,dst_addr,src_addr,proto,af,size,paris_id,result,destination_ip_responded,msm_id,prb_id,timestamp,msm_name,from,type,group_id,stored_timestamp +5040,2.5.1,15,1697941026,185.199.108.153,185.199.108.153,172.17.0.2,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '172.17.0.1', 'ttl': 64, 'size': 76, 'rtt': 0.076}, {'from': '172.17.0.1', 'ttl': 64, 'size': 76, 'rtt': 0.068}, {'from': '172.17.0.1', 'ttl': 64, 'size': 76, 'rtt': 0.027}]}, {'hop': 2, 'result': [{'from': '140.91.198.105', 'ttl': 253, 'size': 28, 'rtt': 0.284}, {'from': '140.91.198.105', 'ttl': 253, 'size': 28, 'rtt': 0.209}, {'from': '140.91.198.105', 'ttl': 253, 'size': 28, 'rtt': 0.188}]}, {'hop': 3, 'result': [{'from': '217.243.179.10', 'ttl': 253, 'size': 28, 'rtt': 0.61}, {'from': '217.243.179.10', 'ttl': 253, 'size': 28, 'rtt': 0.634}, {'from': '217.243.179.10', 'ttl': 253, 'size': 28, 'rtt': 0.739}]}, {'hop': 4, 'result': [{'from': '217.243.179.9', 'ttl': 253, 'size': 68, 'rtt': 1.062}, {'from': '217.243.179.9', 'ttl': 253, 'size': 68, 'rtt': 1.748}, {'from': '217.243.179.9', 'ttl': 253, 'size': 68, 'rtt': 1.107}]}, {'hop': 5, 'result': [{'from': '217.0.201.90', 'ttl': 252, 'size': 68, 'rtt': 11.728}, {'from': '217.0.201.90', 'ttl': 252, 'size': 68, 'rtt': 1.411}, {'from': '217.0.201.90', 'ttl': 252, 'size': 68, 'rtt': 1.419}]}, {'hop': 6, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 7, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 8, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 9, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 10, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,1004810,1697940954,Traceroute,152.70.182.142,traceroute,62390085,1697941046 +5080,2.6.2,31,1697941026,185.199.108.153,185.199.108.153,10.0.0.206,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '140.91.200.18', 'ttl': 254, 'size': 28, 'rtt': 0.188}, {'from': '140.91.200.18', 'ttl': 254, 'size': 28, 'rtt': 0.156}, {'from': '140.91.200.18', 'ttl': 254, 'size': 28, 'rtt': 0.143}]}, {'hop': 2, 'result': [{'from': '213.248.69.255', 'ttl': 254, 'size': 28, 'rtt': 0.781}, {'from': '213.248.69.255', 'ttl': 254, 'size': 28, 'rtt': 0.722}, {'from': '213.248.69.255', 'ttl': 254, 'size': 28, 'rtt': 0.525}]}, {'hop': 3, 'result': [{'from': '213.248.69.254', 'ttl': 63, 'size': 76, 'rtt': 0.59}, {'from': '213.248.69.254', 'ttl': 63, 'size': 76, 'rtt': 0.391}, {'from': '213.248.69.254', 'ttl': 63, 'size': 76, 'rtt': 0.497}]}, {'hop': 4, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 5, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 6, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 7, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 8, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,1005028,1697940954,Traceroute,132.145.49.143,traceroute,62390085,1697941112 +5080,2.6.2,27,1697941027,185.199.108.153,185.199.108.153,172.31.46.105,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '52.56.0.29', 'ttl': 255, 'size': 28, 'rtt': 6.151}, {'from': '52.56.0.29', 'ttl': 255, 'size': 28, 'rtt': 6.294}, {'from': '52.56.0.29', 'ttl': 255, 'size': 28, 'rtt': 6.236}]}, {'hop': 2, 'result': [{'from': '100.65.19.96', 'ttl': 254, 'size': 28, 'rtt': 0.934}, {'from': '100.65.19.96', 'ttl': 254, 'size': 28, 'rtt': 6.312}, {'from': '100.65.19.96', 'ttl': 254, 'size': 28, 'rtt': 6.386}]}, {'hop': 3, 'result': [{'from': '100.66.8.220', 'ttl': 253, 'size': 28, 'rtt': 57.349}, {'from': '100.66.8.220', 'ttl': 253, 'size': 28, 'rtt': 4.555}, {'from': '100.66.8.220', 'ttl': 253, 'size': 28, 'rtt': 4.43}]}, {'hop': 4, 'result': [{'from': '100.66.11.202', 'ttl': 252, 'size': 28, 'rtt': 3.061}, {'from': '100.66.11.202', 'ttl': 252, 'size': 28, 'rtt': 6.293}, {'from': '100.66.11.202', 'ttl': 252, 'size': 28, 'rtt': 6.272}]}, {'hop': 5, 'result': [{'from': '100.66.7.203', 'ttl': 251, 'size': 28, 'rtt': 5.389}, {'from': '100.66.7.203', 'ttl': 251, 'size': 28, 'rtt': 6.581}, {'from': '100.66.7.203', 'ttl': 251, 'size': 28, 'rtt': 6.327}]}, {'hop': 6, 'result': [{'from': '100.66.4.93', 'ttl': 250, 'size': 28, 'rtt': 5.247}, {'from': '100.66.4.93', 'ttl': 250, 'size': 28, 'rtt': 2.607}, {'from': '100.66.4.93', 'ttl': 250, 'size': 28, 'rtt': 6.37}]}, {'hop': 7, 'result': [{'from': '100.65.9.33', 'ttl': 248, 'size': 28, 'rtt': 1.312}, {'from': '100.65.9.33', 'ttl': 248, 'size': 28, 'rtt': 1.096}, {'from': '100.65.9.33', 'ttl': 248, 'size': 28, 'rtt': 1.04}]}, {'hop': 8, 'result': [{'from': '15.230.158.27', 'ttl': 248, 'size': 28, 'rtt': 1.377}, {'from': '15.230.158.27', 'ttl': 248, 'size': 28, 'rtt': 1.597}, {'from': '15.230.158.27', 'ttl': 248, 'size': 28, 'rtt': 2.012}]}, {'hop': 9, 'result': [{'from': '240.2.96.14', 'ttl': 245, 'size': 76, 'rtt': 1.231}, {'from': '240.2.96.14', 'ttl': 245, 'size': 76, 'rtt': 1.13}, {'from': '240.2.96.14', 'ttl': 245, 'size': 76, 'rtt': 1.109}]}, {'hop': 10, 'result': [{'from': '242.5.252.5', 'ttl': 244, 'size': 28, 'rtt': 2.001}, {'from': '242.5.252.5', 'ttl': 244, 'size': 28, 'rtt': 1.765}, {'from': '242.5.252.5', 'ttl': 244, 'size': 28, 'rtt': 1.018}]}, {'hop': 11, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 12, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 13, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 14, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 15, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,1005333,1697940954,Traceroute,18.130.222.49,traceroute,62390085,1697941175 +5080,2.6.2,59,1697941026,185.199.108.153,185.199.108.153,194.55.6.103,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '194.55.6.65', 'ttl': 255, 'size': 28, 'rtt': 0.728}, {'from': '194.55.6.65', 'ttl': 255, 'size': 28, 'rtt': 0.422}, {'from': '194.55.6.65', 'ttl': 255, 'size': 28, 'rtt': 0.638}]}, {'hop': 2, 'result': [{'from': '83.1.168.189', 'ttl': 254, 'size': 28, 'rtt': 0.879}, {'from': '83.1.168.189', 'ttl': 254, 'size': 28, 'rtt': 1.551}, {'from': '83.1.168.189', 'ttl': 254, 'size': 28, 'rtt': 0.881}]}, {'hop': 3, 'result': [{'from': '213.25.5.29', 'ttl': 253, 'size': 28, 'rtt': 0.908}, {'from': '213.25.5.29', 'ttl': 253, 'size': 28, 'rtt': 0.949}, {'from': '213.25.5.29', 'ttl': 253, 'size': 28, 'rtt': 0.867}]}, {'hop': 4, 'result': [{'from': '193.251.249.7', 'ttl': 251, 'size': 28, 'rtt': 22.662}, {'from': '193.251.249.7', 'ttl': 251, 'size': 28, 'rtt': 25.472}, {'from': '193.251.249.7', 'ttl': 251, 'size': 28, 'rtt': 24.015}]}, {'hop': 5, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 6, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 7, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 8, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 9, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,1005852,1697940954,Traceroute,194.55.6.103,traceroute,62390085,1697941045 +5080,2.6.2,62,1697941028,185.199.108.153,185.199.108.153,10.2.0.179,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '10.2.0.1', 'ttl': 64, 'size': 76, 'rtt': 0.542}, {'from': '10.2.0.1', 'ttl': 64, 'size': 76, 'rtt': 0.472}, {'from': '10.2.0.1', 'ttl': 64, 'size': 76, 'rtt': 0.368}]}, {'hop': 2, 'result': [{'from': '37.193.22.254', 'ttl': 254, 'size': 28, 'rtt': 2.438}, {'from': '37.193.22.254', 'ttl': 254, 'size': 28, 'rtt': 1.662}, {'from': '37.193.22.254', 'ttl': 254, 'size': 28, 'rtt': 2.766}]}, {'hop': 3, 'result': [{'from': '10.245.138.253', 'ttl': 253, 'size': 28, 'rtt': 1.574}, {'from': '10.245.138.253', 'ttl': 253, 'size': 28, 'rtt': 1.155}, {'from': '10.245.138.253', 'ttl': 253, 'size': 28, 'rtt': 1.076}]}, {'hop': 4, 'result': [{'from': '10.245.138.254', 'ttl': 252, 'size': 28, 'rtt': 1.493, 'itos': 32}, {'from': '10.245.138.254', 'ttl': 252, 'size': 28, 'rtt': 1.741, 'itos': 32}, {'from': '10.245.138.254', 'ttl': 252, 'size': 28, 'rtt': 1.041, 'itos': 32}]}, {'hop': 5, 'result': [{'from': '178.49.128.6', 'ttl': 251, 'size': 28, 'rtt': 1.545, 'itos': 32}, {'from': '178.49.128.6', 'ttl': 251, 'size': 28, 'rtt': 1.337, 'itos': 32}, {'from': '178.49.128.6', 'ttl': 251, 'size': 28, 'rtt': 1.162, 'itos': 32}]}, {'hop': 6, 'result': [{'from': '87.245.228.193', 'ttl': 241, 'size': 28, 'rtt': 58.644, 'itos': 32}, {'from': '87.245.228.193', 'ttl': 241, 'size': 28, 'rtt': 58.333, 'itos': 32}, {'from': '87.245.228.193', 'ttl': 241, 'size': 28, 'rtt': 58.044, 'itos': 32}]}, {'hop': 7, 'result': [{'from': '87.245.228.192', 'ttl': 245, 'size': 28, 'rtt': 83.122, 'itos': 32}, {'from': '87.245.228.192', 'ttl': 245, 'size': 28, 'rtt': 53.312, 'itos': 32}, {'from': '87.245.228.192', 'ttl': 245, 'size': 28, 'rtt': 53.46, 'itos': 32}]}, {'hop': 8, 'result': [{'from': '87.245.232.181', 'ttl': 245, 'size': 28, 'rtt': 80.632}, {'from': '87.245.232.181', 'ttl': 245, 'size': 28, 'rtt': 78.417}, {'from': '87.245.232.181', 'ttl': 245, 'size': 28, 'rtt': 77.153}]}, {'hop': 9, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 10, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 11, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 12, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 13, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,14015,1697940956,Traceroute,37.193.22.28,traceroute,62390085,1697941122 +5080,2.6.2,10,1697941027,185.199.108.153,185.199.108.153,192.168.2.5,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '192.168.1.1', 'ttl': 64, 'size': 76, 'rtt': 0.638}, {'from': '192.168.1.1', 'ttl': 64, 'size': 76, 'rtt': 0.535}, {'from': '192.168.1.1', 'ttl': 64, 'size': 76, 'rtt': 0.516}]}, {'hop': 2, 'result': [{'from': '10.1.10.1', 'ttl': 63, 'size': 76, 'rtt': 2.725}, {'from': '10.1.10.1', 'ttl': 63, 'size': 76, 'rtt': 3.065}, {'from': '10.1.10.1', 'ttl': 63, 'size': 76, 'rtt': 2.418}]}, {'hop': 3, 'result': [{'from': '96.120.8.189', 'ttl': 62, 'size': 28, 'rtt': 13.469, 'ittl': 0, 'itos': 32}, {'from': '96.120.8.189', 'ttl': 62, 'size': 28, 'rtt': 11.801, 'ittl': 0, 'itos': 32}, {'from': '96.120.8.189', 'ttl': 62, 'size': 28, 'rtt': 10.657, 'ittl': 0, 'itos': 32}]}, {'hop': 4, 'result': [{'from': '162.151.71.225', 'ttl': 61, 'size': 76, 'rtt': 10.784, 'itos': 32}, {'from': '162.151.71.225', 'ttl': 61, 'size': 76, 'rtt': 11.711, 'itos': 32}, {'from': '162.151.71.225', 'ttl': 61, 'size': 76, 'rtt': 10.772, 'itos': 32}]}, {'hop': 5, 'result': [{'from': '96.110.26.178', 'ttl': 61, 'size': 76, 'rtt': 12.698, 'itos': 32}, {'from': '96.110.26.178', 'ttl': 61, 'size': 76, 'rtt': 12.842, 'itos': 32}, {'from': '96.110.26.178', 'ttl': 61, 'size': 76, 'rtt': 10.653, 'itos': 32}]}, {'hop': 6, 'result': [{'from': '96.110.26.161', 'ttl': 251, 'size': 68, 'rtt': 17.715, 'itos': 32}, {'from': '96.110.26.161', 'ttl': 251, 'size': 68, 'rtt': 17.782, 'itos': 32}, {'from': '96.110.26.161', 'ttl': 251, 'size': 68, 'rtt': 26.752, 'itos': 32}]}, {'hop': 7, 'result': [{'from': '69.139.168.141', 'ttl': 250, 'size': 68, 'rtt': 24.753, 'itos': 32}, {'from': '69.139.168.141', 'ttl': 250, 'size': 68, 'rtt': 23.736, 'itos': 32}, {'from': '69.139.168.141', 'ttl': 250, 'size': 68, 'rtt': 24.923, 'itos': 32}]}, {'hop': 8, 'result': [{'from': '96.110.42.161', 'ttl': 249, 'size': 68, 'rtt': 24.587, 'itos': 32}, {'from': '96.110.42.161', 'ttl': 249, 'size': 68, 'rtt': 24.837, 'itos': 32}, {'from': '96.110.42.161', 'ttl': 249, 'size': 68, 'rtt': 25.626, 'itos': 32}]}, {'hop': 9, 'result': [{'from': '96.110.38.130', 'ttl': 248, 'size': 68, 'rtt': 24.751, 'itos': 32}, {'from': '96.110.38.130', 'ttl': 248, 'size': 68, 'rtt': 26.831, 'itos': 32}, {'from': '96.110.38.130', 'ttl': 248, 'size': 68, 'rtt': 24.647, 'itos': 32}]}, {'hop': 10, 'result': [{'from': '96.110.39.165', 'ttl': 247, 'size': 68, 'rtt': 30.713, 'itos': 32}, {'from': '96.110.39.165', 'ttl': 247, 'size': 68, 'rtt': 29.787, 'itos': 32}, {'from': '96.110.39.165', 'ttl': 247, 'size': 68, 'rtt': 28.723, 'itos': 32}]}, {'hop': 11, 'result': [{'from': '68.86.84.145', 'ttl': 246, 'size': 68, 'rtt': 31.783, 'itos': 32}, {'from': '68.86.84.145', 'ttl': 246, 'size': 68, 'rtt': 30.716, 'itos': 32}, {'from': '68.86.84.145', 'ttl': 246, 'size': 68, 'rtt': 31.764, 'itos': 32}]}, {'hop': 12, 'result': [{'from': '96.110.34.114', 'ttl': 245, 'size': 68, 'rtt': 30.679, 'itos': 32}, {'from': '96.110.34.114', 'ttl': 245, 'size': 68, 'rtt': 30.942, 'itos': 32}, {'from': '96.110.34.114', 'ttl': 245, 'size': 68, 'rtt': 31.473, 'itos': 32}]}, {'hop': 13, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 14, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 15, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 16, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 17, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,17946,1697940954,Traceroute,73.52.80.84,traceroute,62390085,1697941261 +5080,2.6.2,69,1697941026,185.199.108.153,185.199.108.153,192.168.1.100,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '192.168.1.1', 'ttl': 64, 'size': 64, 'rtt': 1.391}, {'from': '192.168.1.1', 'ttl': 64, 'size': 64, 'rtt': 0.41}, {'from': '192.168.1.1', 'ttl': 64, 'size': 64, 'rtt': 0.328}]}, {'hop': 2, 'result': [{'from': '10.1.1.1', 'ttl': 63, 'size': 76, 'rtt': 0.932}, {'from': '10.1.1.1', 'ttl': 63, 'size': 76, 'rtt': 0.799}, {'from': '10.1.1.1', 'ttl': 63, 'size': 76, 'rtt': 0.646}]}, {'hop': 3, 'result': [{'from': '108.2.171.1', 'ttl': 253, 'size': 28, 'rtt': 2.687}, {'from': '108.2.171.1', 'ttl': 253, 'size': 28, 'rtt': 15.86}, {'from': '108.2.171.1', 'ttl': 253, 'size': 28, 'rtt': 6.76}]}, {'hop': 4, 'result': [{'from': '130.81.219.162', 'ttl': 252, 'size': 68, 'rtt': 5.857}, {'from': '130.81.219.162', 'ttl': 252, 'size': 68, 'rtt': 5.629}, {'from': '130.81.219.162', 'ttl': 252, 'size': 68, 'rtt': 5.76}]}, {'hop': 5, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 6, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 7, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 8, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 9, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,35064,1697940954,Traceroute,108.52.202.199,traceroute,62390085,1697941111 +5080,2.6.2,37,1697941070,185.199.108.153,185.199.108.153,192.168.40.4,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '192.168.40.1', 'ttl': 64, 'size': 76, 'rtt': 3.359}, {'from': '192.168.40.1', 'ttl': 64, 'size': 76, 'rtt': 2.034}, {'from': '192.168.40.1', 'ttl': 64, 'size': 76, 'rtt': 1.718}]}, {'hop': 2, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 3, 'result': [{'from': '10.128.18.85', 'ttl': 62, 'size': 76, 'rtt': 16.417}, {'from': '10.128.18.85', 'ttl': 62, 'size': 76, 'rtt': 6.183}, {'from': '10.128.18.85', 'ttl': 62, 'size': 76, 'rtt': 9.584}]}, {'hop': 4, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 5, 'result': [{'x': '*'}, {'from': '10.128.118.217', 'ttl': 60, 'size': 76, 'rtt': 11.416}, {'from': '10.128.118.217', 'ttl': 60, 'size': 76, 'rtt': 8.648}]}, {'hop': 6, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 7, 'result': [{'from': '152.44.129.1', 'ttl': 248, 'size': 68, 'rtt': 39.557}, {'from': '152.44.129.1', 'ttl': 248, 'size': 68, 'rtt': 40.058}, {'from': '152.44.129.1', 'ttl': 248, 'size': 68, 'rtt': 38.853}]}, {'hop': 8, 'result': [{'from': '174.127.151.10', 'ttl': 248, 'size': 68, 'rtt': 34.747}, {'from': '174.127.151.10', 'ttl': 248, 'size': 68, 'rtt': 34.608}, {'from': '174.127.151.10', 'ttl': 248, 'size': 68, 'rtt': 36.129}]}, {'hop': 9, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 10, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 11, 'result': [{'late': 30, 'from': '192.168.40.1', 'ttl': 64, 'size': 68, 'edst': '193.0.0.164'}, {'late': 29, 'from': '192.168.40.1', 'ttl': 64, 'size': 68, 'edst': '193.0.0.164'}, {'late': 28, 'from': '192.168.40.1', 'ttl': 64, 'size': 68, 'edst': '193.0.0.164'}, {'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 12, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 13, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,51310,1697940957,Traceroute,66.223.208.181,traceroute,62390085,1697941122 +5080,2.6.2,63,1697941028,185.199.108.153,185.199.108.153,192.168.1.181,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '192.168.1.1', 'ttl': 64, 'size': 76, 'rtt': 2.843}, {'from': '192.168.1.1', 'ttl': 64, 'size': 76, 'rtt': 0.521}, {'from': '192.168.1.1', 'ttl': 64, 'size': 76, 'rtt': 0.702}]}, {'hop': 2, 'result': [{'from': '81.6.46.1', 'ttl': 253, 'size': 28, 'rtt': 0.886}, {'from': '81.6.46.1', 'ttl': 253, 'size': 28, 'rtt': 0.84}, {'from': '81.6.46.1', 'ttl': 253, 'size': 28, 'rtt': 0.833}]}, {'hop': 3, 'result': [{'from': '141.195.80.70', 'ttl': 62, 'size': 76, 'rtt': 1.471}, {'from': '141.195.80.70', 'ttl': 62, 'size': 76, 'rtt': 1.44}, {'from': '141.195.80.70', 'ttl': 62, 'size': 76, 'rtt': 1.465}]}, {'hop': 4, 'result': [{'from': '5.180.135.51', 'ttl': 61, 'size': 76, 'rtt': 1.647}, {'from': '5.180.135.51', 'ttl': 61, 'size': 76, 'rtt': 1.468}, {'from': '5.180.135.51', 'ttl': 61, 'size': 76, 'rtt': 1.236}]}, {'hop': 5, 'result': [{'from': '5.180.135.52', 'ttl': 60, 'size': 76, 'rtt': 2.197}, {'from': '5.180.135.52', 'ttl': 60, 'size': 76, 'rtt': 2.161}, {'from': '5.180.135.52', 'ttl': 60, 'size': 76, 'rtt': 1.977}]}, {'hop': 6, 'result': [{'from': '5.180.135.136', 'ttl': 59, 'size': 76, 'rtt': 1.958}, {'from': '5.180.135.136', 'ttl': 59, 'size': 76, 'rtt': 2.217}, {'from': '5.180.135.136', 'ttl': 59, 'size': 76, 'rtt': 12.752}]}, {'hop': 7, 'result': [{'from': '5.180.135.160', 'ttl': 58, 'size': 76, 'rtt': 2.036}, {'from': '5.180.135.160', 'ttl': 58, 'size': 76, 'rtt': 1.941}, {'from': '5.180.135.160', 'ttl': 58, 'size': 76, 'rtt': 1.902}]}, {'hop': 8, 'result': [{'from': '5.180.134.41', 'ttl': 57, 'size': 76, 'rtt': 12.472}, {'from': '5.180.134.41', 'ttl': 57, 'size': 76, 'rtt': 12.565}, {'from': '5.180.134.41', 'ttl': 57, 'size': 76, 'rtt': 12.483}]}, {'hop': 9, 'result': [{'from': '5.180.135.222', 'ttl': 56, 'size': 76, 'rtt': 13.157}, {'from': '5.180.135.222', 'ttl': 56, 'size': 76, 'rtt': 13.498}, {'from': '5.180.135.222', 'ttl': 56, 'size': 76, 'rtt': 12.759}]}, {'hop': 10, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 11, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 12, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 13, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 14, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,61988,1697940956,Traceroute,85.195.255.83,traceroute,62390085,1697941030 +5080,2.6.2,71,1697941030,185.199.108.153,185.199.108.153,192.168.178.20,UDP,4,48,1,"[{'hop': 1, 'result': [{'from': '192.168.178.1', 'ttl': 64, 'size': 76, 'rtt': 1.873}, {'from': '192.168.178.1', 'ttl': 64, 'size': 76, 'rtt': 0.924}, {'from': '192.168.178.1', 'ttl': 64, 'size': 76, 'rtt': 1.809}]}, {'hop': 2, 'result': [{'from': '178.202.24.1', 'ttl': 254, 'size': 28, 'rtt': 15.026}, {'from': '178.202.24.1', 'ttl': 254, 'size': 28, 'rtt': 11.649}, {'from': '178.202.24.1', 'ttl': 254, 'size': 28, 'rtt': 13.687}]}, {'hop': 3, 'result': [{'from': '81.210.149.102', 'ttl': 253, 'size': 28, 'rtt': 12.508}, {'from': '81.210.149.102', 'ttl': 253, 'size': 28, 'rtt': 13.929}, {'from': '81.210.149.102', 'ttl': 253, 'size': 28, 'rtt': 11.708}]}, {'hop': 4, 'result': [{'from': '145.253.48.220', 'ttl': 247, 'size': 140, 'rtt': 12.658, 'icmpext': {'version': 0, 'rfc4884': 1, 'obj': [{'class': 0, 'type': 0}]}}, {'from': '145.253.48.220', 'ttl': 247, 'size': 140, 'rtt': 14.555, 'icmpext': {'version': 0, 'rfc4884': 1, 'obj': [{'class': 0, 'type': 0}]}}, {'from': '145.253.48.220', 'ttl': 247, 'size': 140, 'rtt': 11.64, 'icmpext': {'version': 0, 'rfc4884': 1, 'obj': [{'class': 0, 'type': 0}]}}]}, {'hop': 5, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 6, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 7, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 8, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 9, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}, {'hop': 255, 'result': [{'x': '*'}, {'x': '*'}, {'x': '*'}]}]",False,62390085,62422,1697940957,Traceroute,109.90.40.189,traceroute,62390085,1697941139 diff --git a/tests/test.html b/tests/test.html new file mode 100644 index 0000000..6bdd6e0 --- /dev/null +++ b/tests/test.html @@ -0,0 +1,9 @@ + +
+ \ No newline at end of file diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..b7bf323 --- /dev/null +++ b/tests/test.py @@ -0,0 +1,57 @@ +from requests import request, get,post +import json +import pandas as pd +from os import getenv +from dotenv import load_dotenv +from pandas_helpers import remove_columns + +# load_dotenv() +# measurement = requests.get( +# "https://atlas.ripe.net/api/v2/measurements/62395910/results/?format=json") + +# probe = requests.get( +# "https://atlas.ripe.net/api/v2/probes/14300/?format=json" +# ) +# probe = probe.json() + +# probe_ip = probe['prefix_v4'].split("/")[0] + + +# json_result = measurement.json() +# ip_list = [] +# ip_list.append(probe_ip) + +# for entry in json_result[0]['result']: +# for result in entry['result']: +# if 'from' in result: +# ip_list.append(result['from']) +# break +# geo_location = [] +# for i, ip in enumerate(ip_list): +# try: +# response = requests.get("https://api.ipgeolocation.io/ipgeo?apiKey=" + getenv("IPGEOLOCATION") + "&ip=" + ip + "&output=json",timeout=10) +# geo_location.append(response.json()) +# except requests.Timeout: +# continue + +# for data in geo_location: +# print(data) + +country_data = None + +with open('./countries.json', 'r') as file: + country_data = json.loads(file.read()) + +cdf = pd.DataFrame(country_data) + + + + +cdf_columns = list(cdf.columns) +cdf_columns.remove('commonName') +cdf_columns.insert(0,'commonName') +cdf_columns.remove('officialName') +cdf_columns.insert(1,'officialName') +cdf = cdf[cdf_columns] +cdf.to_csv('./countries.csv',mode='w',index=False) +cdf.to_json('./countries.json',mode='w',index=False,indent=4) \ No newline at end of file