MVP of tidy watcher

Cargo.lock

@@ -5281,6 +5281,7 @@ dependencies = [
  "cargo_metadata",
  "ignore",
  "lazy_static",
+ "md-5",
  "miropt-test-tools",
  "regex",
  "semver",

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -57,9 +57,10 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
+# tidy-ticket-perf-commit
 # rustc-perf version from 2023-05-30
-# Should also be changed in the opt-dist tool for other environments.
 ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1
+# tidy-ticket-perf-commit
 RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \
     unzip perf.zip && \
     mv rustc-perf-$PERF_COMMIT rustc-perf && \

src/tools/opt-dist/src/environment/windows.rs

@@ -39,10 +39,10 @@ impl Environment for WindowsEnvironment {
     }
 
     fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
-        // FIXME: add some mechanism for synchronization of this commit SHA with
-        // Linux (which builds rustc-perf in a Dockerfile)
+        // tidy-ticket-perf-commit
         // rustc-perf version from 2023-05-30
         const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
+        // tidy-ticket-perf-commit
 
         let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
         let client = reqwest::blocking::Client::builder()

src/tools/tidy/Cargo.toml

@@ -14,6 +14,7 @@ walkdir = "2"
 ignore = "0.4.18"
 semver = "1.0"
 termcolor = "1.1.3"
+md-5 = "0.10"
 
 [[bin]]
 name = "rust-tidy"

src/tools/tidy/src/lib.rs

@@ -72,4 +72,5 @@ pub mod ui_tests;
 pub mod unit_tests;
 pub mod unstable_book;
 pub mod walk;
+pub mod watcher;
 pub mod x_version;

src/tools/tidy/src/main.rs

@@ -137,6 +137,8 @@ fn main() {
 
         check!(x_version, &root_path, &cargo);
 
+        check!(watcher, &root_path);
+
         let collected = {
             drain_handles(&mut handles);
 

src/tools/tidy/src/watcher.rs

@@ -0,0 +1,102 @@
+//! Checks that text between tags unchanged, emitting warning otherwise,
+//! allowing asserting that code in different places over codebase is in sync.
+//!
+//! This works via hashing text between tags and saving hash in tidy.
+//!
+//! Usage:
+//!
+//! some.rs:
+//! // tidy-ticket-foo
+//! const FOO: usize = 42;
+//! // tidy-ticket-foo
+//!
+//! some.sh:
+//! # tidy-ticket-foo
+//! export FOO=42
+//! # tidy-ticket-foo
+use md5::{Digest, Md5};
+use std::fs;
+use std::path::Path;
+
+#[cfg(test)]
+mod tests;
+
+/// Return hash for source text between 2 tag occurrence,
+/// ignoring lines where tag written
+///
+/// Expecting:
+/// tag is not multiline
+/// source always have at least 2 occurrence of tag (>2 ignored)
+fn span_hash(source: &str, tag: &str, bad: &mut bool) -> Result<String, ()> {
+    let start_idx = match source.find(tag) {
+        Some(idx) => idx,
+        None => return Err(tidy_error!(bad, "tag {} should exist in provided text", tag)),
+    };
+    let end_idx = {
+        let end = match source[start_idx + tag.len()..].find(tag) {
+            // index from source start
+            Some(idx) => start_idx + tag.len() + idx,
+            None => return Err(tidy_error!(bad, "tag end {} should exist in provided text", tag)),
+        };
+        // second line with tag can contain some other text before tag, ignore it
+        // by finding position of previous line ending
+        //
+        // FIXME: what if line ending is \r\n? In that case \r will be hashed too
+        let offset = source[start_idx..end].rfind('\n').unwrap();
+        start_idx + offset
+    };
+
+    let mut hasher = Md5::new();
+
+    source[start_idx..end_idx]
+        .lines()
+        // skip first line with tag
+        .skip(1)
+        // hash next lines, ignoring end trailing whitespaces
+        .for_each(|line| {
+            let trimmed = line.trim_end();
+            hasher.update(trimmed);
+        });
+    Ok(format!("{:x}", hasher.finalize()))
+}
+
+fn check_entry(entry: &ListEntry<'_>, bad: &mut bool, root_path: &Path) {
+    let file = fs::read_to_string(root_path.join(Path::new(entry.0))).unwrap();
+    let actual_hash = span_hash(&file, entry.2, bad).unwrap();
+    if actual_hash != entry.1 {
+        // Write tidy error description for wather only once.
+        // Will not work if there was previous errors of other types.
+        if *bad == false {
+            tidy_error!(
+                bad,
+                "Mismatched hashes for tidy watcher found.\n\
+                Check src/tools/tidy/src/watcher.rs, find tag/hash in TIDY_WATCH_LIST list \
+                and verify that sources for provided group of tags in sync. If they in sync, update hash."
+            )
+        }
+        tidy_error!(
+            bad,
+            "hash for tag `{}` in path `{}` mismatch:\n  actual: `{}`, expected: `{}`\n",
+            entry.2,
+            entry.0,
+            actual_hash,
+            entry.1
+        );
+    }
+}
+
+/// (path, hash, tag)
+type ListEntry<'a> = (&'a str, &'a str, &'a str);
+
+/// List of tags to watch, along with paths and hashes
+#[rustfmt::skip]
+const TIDY_WATCH_LIST: &[ListEntry<'_>] = &[
+    ("src/tools/opt-dist/src/environment/windows.rs", "dcad53f163a2775164b5d2faaa70b653", "tidy-ticket-perf-commit"),
+    ("src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile", "76c8d9783e38e25a461355f82fcd7955", "tidy-ticket-perf-commit"),
+];
+
+pub fn check(root_path: &Path, bad: &mut bool) {
+    for entry in TIDY_WATCH_LIST {
+        check_entry(entry, bad, root_path);
+    }
+}

src/tools/tidy/src/watcher/tests.rs

@@ -0,0 +1,39 @@
+use super::*;
+
+#[test]
+fn test_span_hash_one_line() {
+    let source = "some text\ntidy-tag\ncheckme=42\ntidy-tag\n";
+    let tag = "tidy-tag";
+    assert_eq!("42258eba764c3f94a24de379e5715dc8", span_hash(source, tag, &mut true).unwrap());
+}
+
+#[test]
+fn test_span_hash_multiple_lines() {
+    let source = "some text\ntidy-tag\ncheckme=42\nother line\ntidy-tag\n";
+    let tag = "tidy-tag";
+    assert_eq!("49cb23dc2032ceea671ca48092750a1c", span_hash(source, tag, &mut true).unwrap());
+}
+
+#[test]
+fn test_span_hash_has_some_text_in_line_with_tag() {
+    let source = "some text\ntidy-tag ignore me\ncheckme=42\nother line\ntidy-tag\n";
+    let tag = "tidy-tag";
+    assert_eq!("49cb23dc2032ceea671ca48092750a1c", span_hash(source, tag, &mut true).unwrap());
+}
+
+#[test]
+fn test_span_hash_has_some_text_in_line_before_second_tag() {
+    let source = r#"
+RUN ./build-clang.sh
+ENV CC=clang CXX=clang++
+# tidy-ticket-perf-commit
+# rustc-perf version from 2023-05-30
+ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1
+# tidy-ticket-perf-commit
+RUN curl -LS -o perf.zip https://github.com/rust-lang/rustc-perf/archive/$PERF_COMMIT.zip && \
+    unzip perf.zip && \
+    mv rustc-perf-$PERF_COMMIT rustc-perf && \
+    rm perf.zip"#;
+    let tag = "tidy-ticket-perf-commit";
+    assert_eq!("76c8d9783e38e25a461355f82fcd7955", span_hash(source, tag, &mut true).unwrap());
+}