diff --git a/pyproject.toml b/pyproject.toml
index 65ebf55..72e4553 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,6 +78,8 @@ extra_checks = true
 
 [tool.ruff]
 src = ["src", "test"]
+
+[tool.ruff.lint]
 select = [
   "E",   # pycodestyle
   "F",   # pyflakes
@@ -115,9 +117,10 @@ ignore = [
   "D100", "D102", "D103", "D104", "D105", "D107",
   "PTH123",
   "TRY003", "TRY301",
+  "UP032",
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "test/**/*.py" = [
   "S101", "S105",
   "D103",
@@ -127,14 +130,14 @@ ignore = [
   "ARG001",
 ]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
-[tool.ruff.flake8-pytest-style]
+[tool.ruff.lint.flake8-pytest-style]
 fixture-parentheses = false
 mark-parentheses = false
 parametrize-names-type = "csv"
 parametrize-values-type = "list"
 
-[tool.ruff.flake8-tidy-imports]
+[tool.ruff.lint.flake8-tidy-imports]
 ban-relative-imports = "parents"
diff --git a/requirements-lint.txt b/requirements-lint.txt
index c06b044..4897de4 100644
--- a/requirements-lint.txt
+++ b/requirements-lint.txt
@@ -1,5 +1,5 @@
-ruff
-black
+ruff==0.3.5
+black==23.9.1
 mypy
 types-requests
 types-PyYAML
diff --git a/src/security_constraints/main.py b/src/security_constraints/main.py
index 12eaa7b..ec3f5d7 100644
--- a/src/security_constraints/main.py
+++ b/src/security_constraints/main.py
@@ -56,7 +56,9 @@ def sort_vulnerabilities(
     vulnerabilities: List[SecurityVulnerability],
 ) -> List[SecurityVulnerability]:
     """Sort vulnerabilities into the order they should appear in the constraints."""
-    return sorted(vulnerabilities, key=lambda v: v.package)
+    sorted_vulnerabilities = sorted(vulnerabilities, key=lambda v: v.identifier)
+    sorted_vulnerabilities.sort(key=lambda v: v.package)
+    return sorted_vulnerabilities
 
 
 def get_safe_version_constraints(
diff --git a/test/test_main.py b/test/test_main.py
index 1d009f5..03f9939 100644
--- a/test/test_main.py
+++ b/test/test_main.py
@@ -329,12 +329,59 @@ def test_filter_vulnerabilities(
             ],
         ),
         ([], []),
+        (
+            [
+                SecurityVulnerability(
+                    name="CVE-2",
+                    identifier="GHSA-2",
+                    package="pybanana",
+                    vulnerable_range="= 2.0",
+                ),
+                SecurityVulnerability(
+                    name="CVE-3U",
+                    identifier="GHSA-3",
+                    package="pypeel",
+                    vulnerable_range="< 3.0dev1",
+                ),
+                SecurityVulnerability(
+                    name="CVE-1",
+                    identifier="GHSA-1",
+                    package="pybanana",
+                    vulnerable_range="= 1.0",
+                ),
+            ],
+            [
+                SecurityVulnerability(
+                    name="CVE-1",
+                    identifier="GHSA-1",
+                    package="pybanana",
+                    vulnerable_range="= 1.0",
+                ),
+                SecurityVulnerability(
+                    name="CVE-2",
+                    identifier="GHSA-2",
+                    package="pybanana",
+                    vulnerable_range="= 2.0",
+                ),
+                SecurityVulnerability(
+                    name="CVE-3U",
+                    identifier="GHSA-3",
+                    package="pypeel",
+                    vulnerable_range="< 3.0dev1",
+                ),
+            ],
+        ),
     ],
+    ids=["sort by package", "empty", "sub-sort by identifier"],
 )
 def test_sort_vulnerabilities(
     vulnerabilities: List[SecurityVulnerability], expected: List[SecurityVulnerability]
 ) -> None:
+    original_vulnerabilities = vulnerabilities.copy()
     assert sort_vulnerabilities(vulnerabilities=vulnerabilities) == expected
+    assert (
+        vulnerabilities == original_vulnerabilities
+    ), "input list was altered in-place"
 
 
 @pytest.mark.parametrize(