diff --git a/tests/utils/fixtures/packets/metadata/invalid/missing_extras.yaml b/tests/utils/fixtures/packets/metadata/invalid/missing_extras.yaml deleted file mode 100644 index ead3cce7f..000000000 --- a/tests/utils/fixtures/packets/metadata/invalid/missing_extras.yaml +++ /dev/null @@ -1,7 +0,0 @@ -version: 1 -type: plugin-packet -generated_at: 1721728796871 -server_version: 9.11.0 -server_id: p7j6wmx6269jan410vjrylfb2u -license_id: 0ubekqbvxkptxoasnq1qdadkz1 -customer_id: jcvj1vkppgc7takujqe4449itu diff --git a/tests/utils/fixtures/packets/support/full.yaml b/tests/utils/fixtures/packets/support/full.yaml new file mode 100644 index 000000000..dec423585 --- /dev/null +++ b/tests/utils/fixtures/packets/support/full.yaml @@ -0,0 +1,37 @@ +license_to: Mattermost +server_os: linux +server_architecture: amd64 +server_version: 8.1.9 +build_hash: 4c83724516242843802cc75840b08ead6afbd37b +database_type: postgres +database_version: "13.10" +database_schema_version: "113" +active_users: 1 +license_supported_users: 200000 +total_channels: 2 +total_posts: 6 +total_teams: 1 +daily_active_users: 1 +monthly_active_users: 1 +websocket_connections: 0 +master_db_connections: 14 +read_db_connections: 0 +inactive_user_count: 0 +elastic_post_indexing_jobs: [] +elastic_post_aggregation_jobs: [] +ldap_sync_jobs: [] +message_export_jobs: [] +data_retention_jobs: [] +compliance_jobs: [] +migration_jobs: +- id: 4555h6cxb38q3rhnyfu95dypxh + type: migrations + priority: 0 + createat: 1723734966121 + startat: 1723734980530 + lastactivityat: 1723734981002 + status: success + progress: 0 + data: + last_done: '{"current_table":"ChannelMembers","last_team_id":"crro7gj13bdzfjm4rmm6ept6sa","last_channel_id":"mpmdxijsftdodkzbehncatthcr","last_user":"wg94o7yd4jyxjbxoihettwgmah"}' + migration_key: migration_advanced_permissions_phase_2 diff --git a/tests/utils/fixtures/packets/support/invalid.yaml b/tests/utils/fixtures/packets/support/invalid.yaml new file mode 100644 index 000000000..ee505d251 --- /dev/null +++ b/tests/utils/fixtures/packets/support/invalid.yaml @@ -0,0 +1,36 @@ +license_to: Mattermost +server_os: linux +server_architecture: amd64 +build_hash: 4c83724516242843802cc75840b08ead6afbd37b +database_type: postgres +database_version: "13.10" +database_schema_version: 113 +active_users: 1 +license_supported_users: 200000 +total_channels: 2 +total_posts: 6 +total_teams: 1 +daily_active_users: 1 +monthly_active_users: 1 +websocket_connections: 0 +master_db_connections: 14 +read_db_connections: 0 +inactive_user_count: 0 +elastic_post_indexing_jobs: [] +elastic_post_aggregation_jobs: [] +ldap_sync_jobs: [] +message_export_jobs: [] +data_retention_jobs: [] +compliance_jobs: [] +migration_jobs: +- id: 4555h6cxb38q3rhnyfu95dypxh + type: migrations + priority: 0 + createat: 1723734966121 + startat: 1723734980530 + lastactivityat: 1723734981002 + status: success + progress: 0 + data: + last_done: '{"current_table":"ChannelMembers","last_team_id":"crro7gj13bdzfjm4rmm6ept6sa","last_channel_id":"mpmdxijsftdodkzbehncatthcr","last_user":"wg94o7yd4jyxjbxoihettwgmah"}' + migration_key: migration_advanced_permissions_phase_2 diff --git a/tests/utils/fixtures/packets/support/valid_with_metadata.zip b/tests/utils/fixtures/packets/support/valid_with_metadata.zip new file mode 100644 index 000000000..313292eaf Binary files /dev/null and b/tests/utils/fixtures/packets/support/valid_with_metadata.zip differ diff --git a/tests/utils/fixtures/packets/support/valid_without_metadata.zip b/tests/utils/fixtures/packets/support/valid_without_metadata.zip new file mode 100644 index 000000000..ef378e44a Binary files /dev/null and b/tests/utils/fixtures/packets/support/valid_without_metadata.zip differ diff --git a/tests/utils/packets/test_loaders.py b/tests/utils/packets/test_loaders.py index 3d8872a90..f4b3eb906 100644 --- a/tests/utils/packets/test_loaders.py +++ b/tests/utils/packets/test_loaders.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import date, datetime, timezone from pathlib import Path import pandas as pd @@ -6,12 +6,20 @@ from pandas.testing import assert_frame_equal from pydantic import ValidationError -from utils.packets.loaders import load_metadata, load_user_survey, load_user_survey_package +from utils.packets.loaders import ( + load_metadata, + load_support_packet_file, + load_support_packet_info, + load_user_survey, + load_user_survey_package, +) from utils.packets.models.metadata import Extras, SupportPacketMetadata, SupportPacketTypeEnum +from utils.packets.models.support import JobV1 FIXTURE_DIR = Path(__file__).parent.parent / 'fixtures' / 'packets' METADATA_DIR = FIXTURE_DIR / 'metadata' SURVEY_DIR = FIXTURE_DIR / 'user_survey' +SUPPORT_DIR = FIXTURE_DIR / 'support' # @@ -78,7 +86,6 @@ def test_load_full_metadata(metadata, expected): [ pytest.param(METADATA_DIR / 'invalid' / 'invalid_timestamp.yaml', [('generated_at',)], id='invalid timestamp'), pytest.param(METADATA_DIR / 'invalid' / 'invalid_type.yaml', [('type',)], id='invalid type'), - pytest.param(METADATA_DIR / 'invalid' / 'missing_extras.yaml', [('extras',)], id='missing extras'), pytest.param( METADATA_DIR / 'invalid' / 'missing_fields.yaml', [('generated_at',), ('server_id',), ('server_version',)], @@ -233,3 +240,109 @@ def test_load_invalid_plugin_id(): # THEN: expect proper issue to be raised assert str(exc.value) == 'Not a user survey packet - packet type is com.mattermost.plugin' + + +# +# Support packet loader tests +# + + +def test_load_support_packet_full(): + # WHEN: attempt to load a full support packet + with open(SUPPORT_DIR / 'full.yaml', 'r') as fp: + sp = load_support_packet_info(fp) + + # THEN: expect support packet to be loaded correctly + assert sp.license_to == 'Mattermost' + assert sp.server_os == 'linux' + assert sp.server_architecture == 'amd64' + assert sp.build_hash == '4c83724516242843802cc75840b08ead6afbd37b' + assert sp.database_type == 'postgres' + assert sp.database_version == '13.10' + assert sp.database_schema_version == '113' + assert sp.active_users == 1 + assert sp.license_supported_users == 200000 + assert sp.total_channels == 2 + assert sp.total_posts == 6 + assert sp.total_teams == 1 + assert sp.daily_active_users == 1 + assert sp.monthly_active_users == 1 + assert sp.websocket_connections == 0 + assert sp.master_db_connections == 14 + assert sp.read_db_connections == 0 + assert sp.inactive_user_count == 0 + assert sp.elastic_post_indexing_jobs == [] + assert sp.elastic_post_aggregation_jobs == [] + assert sp.ldap_sync_jobs == [] + assert sp.message_export_jobs == [] + assert sp.data_retention_jobs == [] + assert sp.compliance_jobs == [] + assert sp.bleve_post_indexing_jobs is None + assert sp.migration_jobs == [ + JobV1( + id='4555h6cxb38q3rhnyfu95dypxh', + type='migrations', + priority=0, + createat=datetime(2024, 8, 15, 15, 16, 6, 121000, timezone.utc), + startat=datetime(2024, 8, 15, 15, 16, 20, 530000, timezone.utc), + lastactivityat=datetime(2024, 8, 15, 15, 16, 21, 2000, timezone.utc), + status='success', + progress=0, + data={ + 'last_done': '{"current_table":"ChannelMembers","last_team_id":"crro7gj13bdzfjm4rmm6ept6sa","last_channel_id":"mpmdxijsftdodkzbehncatthcr","last_user":"wg94o7yd4jyxjbxoihettwgmah"}', # noqa: E501 + 'migration_key': 'migration_advanced_permissions_phase_2', + }, + ) + ] + + +def test_support_packet_invalid(): + # WHEN: attempt to load an invalid support packet + with pytest.raises(ValidationError) as exc, open(SUPPORT_DIR / 'invalid.yaml', 'r') as fp: + load_support_packet_info(fp) + + assert sorted([e['loc'] for e in exc.value.errors()]) == [ + ('database_schema_version',), # Int instead of string + ('server_version',), # Missing + ] + + +# +# Full loader for support packet v1 +# + + +def test_load_full_support_packet_v1_with_metadata(): + # WHEN: attempt to load a valid support packet + metadata, sp = load_support_packet_file(SUPPORT_DIR / 'valid_with_metadata.zip') + + # THEN: expect metadata to be loaded correctly + assert metadata == SupportPacketMetadata( + version=1, + type=SupportPacketTypeEnum.support_packet, + generated_at=datetime(2024, 8, 19, 13, 46, 45, 94000, tzinfo=timezone.utc), + server_version='9.11.0', + server_id='rmg9ib5rspy93jxswyc454bwzo', + license_id='mud3ihm4938dxncqasxt14xxch', + customer_id='p9un369a67gimj4yd6i6ib39wh', + ) + + # THEN: expect responses to be loaded correctly + assert sp.license_to == 'Mattermost' + assert sp.server_os == 'linux' + assert sp.server_architecture == 'amd64' + assert sp.build_hash == '0bc2ddd42375a75ab14e63f038165150d4f07659' + + +def test_load_full_support_packet_v1_without_metadata(): + # WHEN: attempt to load a valid support packet + metadata, sp = load_support_packet_file(SUPPORT_DIR / 'valid_without_metadata.zip') + + # THEN: expect metadata to be loaded correctly + assert metadata is None + + # THEN: expect responses to be loaded correctly + assert sp.license_to == 'Mattermost' + assert sp.server_os == 'linux' + assert sp.server_architecture == 'amd64' + assert sp.build_hash == '4c83724516242843802cc75840b08ead6afbd37b' diff --git a/utils/packets/__main__.py b/utils/packets/__main__.py index 8c4213765..aacf7cc0b 100644 --- a/utils/packets/__main__.py +++ b/utils/packets/__main__.py @@ -3,7 +3,7 @@ import click from utils.helpers import initialize_cli_logging -from utils.packets.service import ingest_survey_packet +from utils.packets.service import ingest_support_packet, ingest_survey_packet initialize_cli_logging(logging.INFO, 'stderr') @@ -23,7 +23,19 @@ def user_survey( ) -> None: """ Ingest a user survey packet. - :param input: The zip file with the user survey packet data. """ ingest_survey_packet(input) + + +@packets.command() +@click.argument('input', type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True)) +def support_v1( + input: click.Path, +) -> None: + """ + Ingest a support packet using the original support package specification. + + :param input: The zip file with the support package. + """ + ingest_support_packet(input) diff --git a/utils/packets/loaders.py b/utils/packets/loaders.py index 1384ccfd5..126691dd5 100644 --- a/utils/packets/loaders.py +++ b/utils/packets/loaders.py @@ -8,9 +8,11 @@ import yaml from utils.packets.models.metadata import SupportPacketMetadata +from utils.packets.models.support import SupportPacketV1 from utils.packets.models.user_survey import UserSurveyMetadata SUPPORT_PACKET_METADATA_FILE = 'metadata.yaml' +SUPPORT_PACKET_FILE = 'support_packet.yaml' SURVEY_METADATA_FILE = 'survey_metadata.json' SURVEY_DATA_FILE = 'responses.csv' @@ -80,3 +82,29 @@ def load_user_survey_package(user_survey_zip_file: str | os.PathLike) -> Tuple[S survey_data = load_user_survey(survey_metadata_fp, survey_data_fp) return metadata, survey_data + + +def load_support_packet_info(metadata_file: IO) -> SupportPacketV1: + """ + Load support packet from a YAML file. + """ + data = yaml.safe_load(metadata_file) + return SupportPacketV1(**data) + + +def load_support_packet_file( + support_packet_zip_file: str | os.PathLike, +) -> Tuple[SupportPacketMetadata, SupportPacketV1]: + with ZipFile(support_packet_zip_file, 'r') as zipfile: + + if SUPPORT_PACKET_METADATA_FILE in zipfile.namelist(): + # Metadata might not be present in older versions of the support packet + with zipfile.open(SUPPORT_PACKET_METADATA_FILE) as metadata_fp: + metadata = load_metadata(metadata_fp) + else: + metadata = None + + with zipfile.open(SUPPORT_PACKET_FILE) as packet_fp: + packet = load_support_packet_info(packet_fp) + + return metadata, packet diff --git a/utils/packets/models/metadata.py b/utils/packets/models/metadata.py index dbbb2de11..fc0a7ca93 100644 --- a/utils/packets/models/metadata.py +++ b/utils/packets/models/metadata.py @@ -24,4 +24,4 @@ class SupportPacketMetadata(BaseModel, extra='ignore'): server_id: str = Field(min_length=26, max_length=26) license_id: str | None = Field(None, min_length=0, max_length=26) customer_id: str | None = Field(None, min_length=0, max_length=26) - extras: Extras + extras: Extras | None = Field(None) diff --git a/utils/packets/models/support.py b/utils/packets/models/support.py new file mode 100644 index 000000000..83c5dff6e --- /dev/null +++ b/utils/packets/models/support.py @@ -0,0 +1,117 @@ +from datetime import datetime +from enum import Enum +from typing import List + +from pydantic import BaseModel, Field + + +class JobType(str, Enum): + data_retention = "data_retention" + message_export = "message_export" + elasticsearch_post_indexing = "elasticsearch_post_indexing" + elasticsearch_post_aggregation = "elasticsearch_post_aggregation" + bleve_post_indexing = "bleve_post_indexing" + ldap_sync = "ldap_sync" + migrations = "migrations" + plugins = "plugins" + expiry_notify = "expiry_notify" + product_notices = "product_notices" + active_users = "active_users" + import_process = "import_process" + import_delete = "import_delete" + export_process = "export_process" + export_delete = "export_delete" + cloud = "cloud" + resend_invitation_email = "resend_invitation_email" + extract_content = "extract_content" + last_accessible_post = "last_accessible_post" + last_accessible_file = "last_accessible_file" + upgrade_notify_admin = "upgrade_notify_admin" + trial_notify_admin = "trial_notify_admin" + post_persistent_notifications = "post_persistent_notifications" + install_plugin_notify_admin = "install_plugin_notify_admin" + hosted_purchase_screening = "hosted_purchase_screening" + s3_path_migration = "s3_path_migration" + cleanup_desktop_tokens = "cleanup_desktop_tokens" + delete_empty_drafts_migration = "delete_empty_drafts_migration" + refresh_post_stats = "refresh_post_stats" + delete_orphan_drafts_migration = "delete_orphan_drafts_migration" + export_users_to_csv = "export_users_to_csv" + delete_dms_preferences_migration = "delete_dms_preferences_migration" + + +class JobStatus(str, Enum): + pending = "pending" + in_progress = "in_progress" + success = "success" + error = "error" + cancel_requested = "cancel_requested" + canceled = "canceled" + warning = "warning" + + +class JobV1(BaseModel): + id: str + type: JobType + priority: int + createat: datetime + startat: datetime + lastactivityat: datetime + status: JobStatus + progress: int + data: dict + + +class SupportPacketV1(BaseModel): + # Based on https://github.com/mattermost/mattermost/blob/master/server/public/model/support_packet.go#L15 + server_os: str = Field(description="The operating system of the server") + server_architecture: str = Field(description="The architecture of the server, i.e. amd64") + server_version: str = Field(description="The version of the Mattermost server") + build_hash: str = Field(description="The build hash of the Mattermost server") + + # DB + database_type: str = Field(description="The type of database being used, i.e. postgres or mysql") + database_version: str = Field(description="The version of the database being used") + database_schema_version: str = Field(description="The schema version of the database being used") + websocket_connections: int = Field(description="The number of websocket connections") + master_db_connections: int = Field(description="The number of master database connections") + read_db_connections: int = Field(description="The number of read database connections") + + # Cluster + cluster_id: str | None = Field(None, description="The cluster ID, if server is running in cluster mode") + + # File store + file_driver: str | None = Field(None, description="The file store driver being used, i.e. local or s3") + file_status: str | None = Field(None, description="The status of the file store") + + # LDAP + ldap_vendor_name: str | None = Field(None) + ldap_verndor_version: str | None = Field(None) + + # ElasticSearch + elastic_server_version: str | None = Field(None) + elastic_server_plugins: str | None = Field(None) + + # License + license_to: str = Field(description="The name of the license owner, extracted from the license.") + license_supported_users: int = Field(description="The number of supported users in the license.") + license_is_trial: bool | None = Field(None, description="Whether the license is a trial license.") + + # Server Stats + active_users: int = Field(description="The number of unique active users") + daily_active_users: int = Field(description="The number of daily active users") + monthly_active_users: int = Field(description="The number of monthly active users") + inactive_user_count: int = Field(description="The number of inactive users") + total_posts: int = Field(description="The total number of posts") + total_channels: int = Field(description="The total number of channels") + total_teams: int = Field(description="The total number of teams") + + # Jobs + data_retention_jobs: List[JobV1] | None = Field(None, description="Data retention jobs") + bleve_post_indexing_jobs: List[JobV1] | None = Field(None, description="Bleve post indexing jobs") + message_export_jobs: List[JobV1] | None = Field(None, description="Message export jobs") + elastic_post_indexing_jobs: List[JobV1] | None = Field(None, description="Bleve post indexing jobs") + elastic_post_aggregation_jobs: List[JobV1] | None = Field(None, description="Elasticsearch post aggregation jobs") + ldap_sync_jobs: List[JobV1] | None = Field(None, description="LDAP sync jobs") + migration_jobs: List[JobV1] | None = Field(None, description="Migration jobs") + compliance_jobs: List[JobV1] | None = Field(None, description="Compliance jobs") diff --git a/utils/packets/service.py b/utils/packets/service.py index d04a91201..7f6961207 100644 --- a/utils/packets/service.py +++ b/utils/packets/service.py @@ -3,7 +3,7 @@ from click import ClickException -from utils.packets.loaders import UserSurveyFixedColumns, load_user_survey_package +from utils.packets.loaders import UserSurveyFixedColumns, load_support_packet_file, load_user_survey_package logger = getLogger(__name__) @@ -19,9 +19,32 @@ def ingest_survey_packet(survey_packet: str | os.PathLike): logger.info('Loaded survey packet') logger.info(f' -> Server ID: {metadata.server_id}') logger.info(f' -> License ID: {metadata.license_id}') - logger.info('\n') + logger.info('') logger.info(f' Total {df[UserSurveyFixedColumns.user_id.value].nunique()} unique users') # TODO: ingest in database except ValueError as e: raise ClickException(f'Error loading survey packet: {e}') + + +def ingest_support_packet(support_packet: str | os.PathLike): + """ + Load support package data and metadata. + + :support_package: The path to the survey packet file. + """ + try: + metadata, sp = load_support_packet_file(support_packet) + logger.info('Loaded support packet') + if metadata: + logger.info(f' -> Server ID: {metadata.server_id}') + logger.info(f' -> License ID: {metadata.license_id}') + else: + logger.info('Support packet does not include metadata file') + logger.info('') + logger.info(f' Server: {sp.server_version} ({sp.server_os} {sp.server_architecture})') + logger.info(f' Database: {sp.database_type} {sp.database_version} (schema {sp.database_schema_version})') + + # TODO: ingest in database + except ValueError as e: + raise ClickException(f'Error loading support package: {e}')