MM-60179: add support package (#1580)

* MM-60179: add support package

* Fix linter issues, revert user survey

* Fix linter issues

* Rename support package to support packet

tests/utils/fixtures/packets/support/full.yaml

@@ -0,0 +1,37 @@
+license_to: Mattermost
+server_os: linux
+server_architecture: amd64
+server_version: 8.1.9
+build_hash: 4c83724516242843802cc75840b08ead6afbd37b
+database_type: postgres
+database_version: "13.10"
+database_schema_version: "113"
+active_users: 1
+license_supported_users: 200000
+total_channels: 2
+total_posts: 6
+total_teams: 1
+daily_active_users: 1
+monthly_active_users: 1
+websocket_connections: 0
+master_db_connections: 14
+read_db_connections: 0
+inactive_user_count: 0
+elastic_post_indexing_jobs: []
+elastic_post_aggregation_jobs: []
+ldap_sync_jobs: []
+message_export_jobs: []
+data_retention_jobs: []
+compliance_jobs: []
+migration_jobs:
+- id: 4555h6cxb38q3rhnyfu95dypxh
+  type: migrations
+  priority: 0
+  createat: 1723734966121
+  startat: 1723734980530
+  lastactivityat: 1723734981002
+  status: success
+  progress: 0
+  data:
+    last_done: '{"current_table":"ChannelMembers","last_team_id":"crro7gj13bdzfjm4rmm6ept6sa","last_channel_id":"mpmdxijsftdodkzbehncatthcr","last_user":"wg94o7yd4jyxjbxoihettwgmah"}'
+    migration_key: migration_advanced_permissions_phase_2

tests/utils/fixtures/packets/support/invalid.yaml

@@ -0,0 +1,36 @@
+license_to: Mattermost
+server_os: linux
+server_architecture: amd64
+build_hash: 4c83724516242843802cc75840b08ead6afbd37b
+database_type: postgres
+database_version: "13.10"
+database_schema_version: 113
+active_users: 1
+license_supported_users: 200000
+total_channels: 2
+total_posts: 6
+total_teams: 1
+daily_active_users: 1
+monthly_active_users: 1
+websocket_connections: 0
+master_db_connections: 14
+read_db_connections: 0
+inactive_user_count: 0
+elastic_post_indexing_jobs: []
+elastic_post_aggregation_jobs: []
+ldap_sync_jobs: []
+message_export_jobs: []
+data_retention_jobs: []
+compliance_jobs: []
+migration_jobs:
+- id: 4555h6cxb38q3rhnyfu95dypxh
+  type: migrations
+  priority: 0
+  createat: 1723734966121
+  startat: 1723734980530
+  lastactivityat: 1723734981002
+  status: success
+  progress: 0
+  data:
+    last_done: '{"current_table":"ChannelMembers","last_team_id":"crro7gj13bdzfjm4rmm6ept6sa","last_channel_id":"mpmdxijsftdodkzbehncatthcr","last_user":"wg94o7yd4jyxjbxoihettwgmah"}'
+    migration_key: migration_advanced_permissions_phase_2

tests/utils/packets/test_loaders.py

@@ -1,17 +1,25 @@
-from datetime import date
+from datetime import date, datetime, timezone
 from pathlib import Path
 
 import pandas as pd
 import pytest
 from pandas.testing import assert_frame_equal
 from pydantic import ValidationError
 
-from utils.packets.loaders import load_metadata, load_user_survey, load_user_survey_package
+from utils.packets.loaders import (
+    load_metadata,
+    load_support_packet_file,
+    load_support_packet_info,
+    load_user_survey,
+    load_user_survey_package,
+)
 from utils.packets.models.metadata import Extras, SupportPacketMetadata, SupportPacketTypeEnum
+from utils.packets.models.support import JobV1
 
 FIXTURE_DIR = Path(__file__).parent.parent / 'fixtures' / 'packets'
 METADATA_DIR = FIXTURE_DIR / 'metadata'
 SURVEY_DIR = FIXTURE_DIR / 'user_survey'
+SUPPORT_DIR = FIXTURE_DIR / 'support'
 
 
 #
@@ -78,7 +86,6 @@ def test_load_full_metadata(metadata, expected):
     [
         pytest.param(METADATA_DIR / 'invalid' / 'invalid_timestamp.yaml', [('generated_at',)], id='invalid timestamp'),
         pytest.param(METADATA_DIR / 'invalid' / 'invalid_type.yaml', [('type',)], id='invalid type'),
-        pytest.param(METADATA_DIR / 'invalid' / 'missing_extras.yaml', [('extras',)], id='missing extras'),
         pytest.param(
             METADATA_DIR / 'invalid' / 'missing_fields.yaml',
             [('generated_at',), ('server_id',), ('server_version',)],
@@ -233,3 +240,109 @@ def test_load_invalid_plugin_id():
 
     # THEN: expect proper issue to be raised
     assert str(exc.value) == 'Not a user survey packet - packet type is com.mattermost.plugin'
+
+
+#
+# Support packet loader tests
+#
+
+
+def test_load_support_packet_full():
+    # WHEN: attempt to load a full support packet
+    with open(SUPPORT_DIR / 'full.yaml', 'r') as fp:
+        sp = load_support_packet_info(fp)
+
+    # THEN: expect support packet to be loaded correctly
+    assert sp.license_to == 'Mattermost'
+    assert sp.server_os == 'linux'
+    assert sp.server_architecture == 'amd64'
+    assert sp.build_hash == '4c83724516242843802cc75840b08ead6afbd37b'
+    assert sp.database_type == 'postgres'
+    assert sp.database_version == '13.10'
+    assert sp.database_schema_version == '113'
+    assert sp.active_users == 1
+    assert sp.license_supported_users == 200000
+    assert sp.total_channels == 2
+    assert sp.total_posts == 6
+    assert sp.total_teams == 1
+    assert sp.daily_active_users == 1
+    assert sp.monthly_active_users == 1
+    assert sp.websocket_connections == 0
+    assert sp.master_db_connections == 14
+    assert sp.read_db_connections == 0
+    assert sp.inactive_user_count == 0
+    assert sp.elastic_post_indexing_jobs == []
+    assert sp.elastic_post_aggregation_jobs == []
+    assert sp.ldap_sync_jobs == []
+    assert sp.message_export_jobs == []
+    assert sp.data_retention_jobs == []
+    assert sp.compliance_jobs == []
+    assert sp.bleve_post_indexing_jobs is None
+    assert sp.migration_jobs == [
+        JobV1(
+            id='4555h6cxb38q3rhnyfu95dypxh',
+            type='migrations',
+            priority=0,
+            createat=datetime(2024, 8, 15, 15, 16, 6, 121000, timezone.utc),
+            startat=datetime(2024, 8, 15, 15, 16, 20, 530000, timezone.utc),
+            lastactivityat=datetime(2024, 8, 15, 15, 16, 21, 2000, timezone.utc),
+            status='success',
+            progress=0,
+            data={
+                'last_done': '{"current_table":"ChannelMembers","last_team_id":"crro7gj13bdzfjm4rmm6ept6sa","last_channel_id":"mpmdxijsftdodkzbehncatthcr","last_user":"wg94o7yd4jyxjbxoihettwgmah"}',  # noqa: E501
+                'migration_key': 'migration_advanced_permissions_phase_2',
+            },
+        )
+    ]
+
+
+def test_support_packet_invalid():
+    # WHEN: attempt to load an invalid support packet
+    with pytest.raises(ValidationError) as exc, open(SUPPORT_DIR / 'invalid.yaml', 'r') as fp:
+        load_support_packet_info(fp)
+
+    assert sorted([e['loc'] for e in exc.value.errors()]) == [
+        ('database_schema_version',),  # Int instead of string
+        ('server_version',),  # Missing
+    ]
+
+
+#
+# Full loader for support packet v1
+#
+
+
+def test_load_full_support_packet_v1_with_metadata():
+    # WHEN: attempt to load a valid support packet
+    metadata, sp = load_support_packet_file(SUPPORT_DIR / 'valid_with_metadata.zip')
+
+    # THEN: expect metadata to be loaded correctly
+    assert metadata == SupportPacketMetadata(
+        version=1,
+        type=SupportPacketTypeEnum.support_packet,
+        generated_at=datetime(2024, 8, 19, 13, 46, 45, 94000, tzinfo=timezone.utc),
+        server_version='9.11.0',
+        server_id='rmg9ib5rspy93jxswyc454bwzo',
+        license_id='mud3ihm4938dxncqasxt14xxch',
+        customer_id='p9un369a67gimj4yd6i6ib39wh',
+    )
+
+    # THEN: expect responses to be loaded correctly
+    assert sp.license_to == 'Mattermost'
+    assert sp.server_os == 'linux'
+    assert sp.server_architecture == 'amd64'
+    assert sp.build_hash == '0bc2ddd42375a75ab14e63f038165150d4f07659'
+
+
+def test_load_full_support_packet_v1_without_metadata():
+    # WHEN: attempt to load a valid support packet
+    metadata, sp = load_support_packet_file(SUPPORT_DIR / 'valid_without_metadata.zip')
+
+    # THEN: expect metadata to be loaded correctly
+    assert metadata is None
+
+    # THEN: expect responses to be loaded correctly
+    assert sp.license_to == 'Mattermost'
+    assert sp.server_os == 'linux'
+    assert sp.server_architecture == 'amd64'
+    assert sp.build_hash == '4c83724516242843802cc75840b08ead6afbd37b'

utils/packets/__main__.py

@@ -3,7 +3,7 @@
 import click
 
 from utils.helpers import initialize_cli_logging
-from utils.packets.service import ingest_survey_packet
+from utils.packets.service import ingest_support_packet, ingest_survey_packet
 
 initialize_cli_logging(logging.INFO, 'stderr')
 
@@ -23,7 +23,19 @@ def user_survey(
 ) -> None:
     """
     Ingest a user survey packet.
-
     :param input: The zip file with the user survey packet data.
     """
     ingest_survey_packet(input)
+
+
+@packets.command()
+@click.argument('input', type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True))
+def support_v1(
+    input: click.Path,
+) -> None:
+    """
+    Ingest a support packet using the original support package specification.
+
+    :param input: The zip file with the support package.
+    """
+    ingest_support_packet(input)

utils/packets/loaders.py

@@ -8,9 +8,11 @@
 import yaml
 
 from utils.packets.models.metadata import SupportPacketMetadata
+from utils.packets.models.support import SupportPacketV1
 from utils.packets.models.user_survey import UserSurveyMetadata
 
 SUPPORT_PACKET_METADATA_FILE = 'metadata.yaml'
+SUPPORT_PACKET_FILE = 'support_packet.yaml'
 SURVEY_METADATA_FILE = 'survey_metadata.json'
 SURVEY_DATA_FILE = 'responses.csv'
 
@@ -80,3 +82,29 @@ def load_user_survey_package(user_survey_zip_file: str | os.PathLike) -> Tuple[S
             survey_data = load_user_survey(survey_metadata_fp, survey_data_fp)
 
     return metadata, survey_data
+
+
+def load_support_packet_info(metadata_file: IO) -> SupportPacketV1:
+    """
+    Load support packet from a YAML file.
+    """
+    data = yaml.safe_load(metadata_file)
+    return SupportPacketV1(**data)
+
+
+def load_support_packet_file(
+    support_packet_zip_file: str | os.PathLike,
+) -> Tuple[SupportPacketMetadata, SupportPacketV1]:
+    with ZipFile(support_packet_zip_file, 'r') as zipfile:
+
+        if SUPPORT_PACKET_METADATA_FILE in zipfile.namelist():
+            # Metadata might not be present in older versions of the support packet
+            with zipfile.open(SUPPORT_PACKET_METADATA_FILE) as metadata_fp:
+                metadata = load_metadata(metadata_fp)
+        else:
+            metadata = None
+
+        with zipfile.open(SUPPORT_PACKET_FILE) as packet_fp:
+            packet = load_support_packet_info(packet_fp)
+
+    return metadata, packet

utils/packets/models/metadata.py

@@ -24,4 +24,4 @@ class SupportPacketMetadata(BaseModel, extra='ignore'):
     server_id: str = Field(min_length=26, max_length=26)
     license_id: str | None = Field(None, min_length=0, max_length=26)
     customer_id: str | None = Field(None, min_length=0, max_length=26)
-    extras: Extras
+    extras: Extras | None = Field(None)