Swap compile time tokens to embedded runtime decoding (#30)

Swap compile time tokens to embedded runtime decoding

Greatly helps with compile times.

.github/workflows/ci.yml

@@ -47,40 +47,11 @@ jobs:
         path: assets/saves
         key: assets/saves
 
-    - name: Set cross environment variables
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        echo "EU4_IRONMAN_TOKENS=/project/assets/tokens/eu4.txt" >> $GITHUB_ENV
-        echo "CK3_IRONMAN_TOKENS=/project/assets/tokens/ck3.txt" >> $GITHUB_ENV
-        echo "IMPERATOR_TOKENS=/project/assets/tokens/imperator.txt" >> $GITHUB_ENV
-        echo "HOI4_IRONMAN_TOKENS=/project/assets/tokens/hoi4.txt" >> $GITHUB_ENV
-        echo "VIC3_IRONMAN_TOKENS=/project/assets/tokens/vic3.txt" >> $GITHUB_ENV
-
-    - name: Set cargo environment variables
-      if: matrix.os != 'ubuntu-latest'
-      shell: bash
-      run: |
-        echo "EU4_IRONMAN_TOKENS=$current_dir/assets/tokens/eu4.txt" >> $GITHUB_ENV
-        echo "CK3_IRONMAN_TOKENS=$current_dir/assets/tokens/ck3.txt" >> $GITHUB_ENV
-        echo "IMPERATOR_TOKENS=$current_dir/assets/tokens/imperator.txt" >> $GITHUB_ENV
-        echo "HOI4_IRONMAN_TOKENS=$current_dir/assets/tokens/hoi4.txt" >> $GITHUB_ENV
-        echo "VIC3_IRONMAN_TOKENS=$current_dir/assets/tokens/vic3.txt" >> $GITHUB_ENV
-      env:
-        current_dir: ${{ github.workspace }}
-
     - name: Install Cross
       if: matrix.os == 'ubuntu-latest'
       run: |
         cargo install --version 0.2.1 cross
         echo "CARGO=cross" >> $GITHUB_ENV
-    - name: Check work
-      run: |
-        echo "CARGO=${{ env.CARGO }}"
-        echo "EU4_IRONMAN_TOKENS=${{ env.EU4_IRONMAN_TOKENS }}"
-        echo "CK3_IRONMAN_TOKENS=${{ env.CK3_IRONMAN_TOKENS }}"
-        echo "IMPERATOR_TOKENS=${{ env.IMPERATOR_TOKENS }}"
-        echo "HOI4_IRONMAN_TOKENS=${{ env.HOI4_IRONMAN_TOKENS }}"
-        echo "VIC3_IRONMAN_TOKENS=${{ env.VIC3_IRONMAN_TOKENS }}"
     - name: Build
       run: ${{ env.CARGO }} build --verbose --target "${{ matrix.build }}"
     - name: Test

.gitignore

@@ -1,4 +1,4 @@
 /target
 assets/saves
-assets/tokens/*.txt
+assets/tokens
 .idea

Cargo.toml

@@ -6,7 +6,7 @@ edition = "2018"
 publish = false
 
 [dependencies]
-jomini = "0.26"
+jomini = "0.27"
 eu4save = { git = "https://github.com/rakaly/eu4save.git", default-features = false, features = ["libdeflate"] }
 imperator-save = { git = "https://github.com/rakaly/imperator-save.git", default-features = false, features = ["libdeflate"] }
 ck3save = { git = "https://github.com/rakaly/ck3save.git", default-features = false, features = ["libdeflate"] }

build.rs

@@ -0,0 +1,10 @@
+fn main() {
+    std::fs::create_dir_all("assets/tokens").expect("to create tokens directory");
+    for game in ["ck3", "hoi4", "eu4", "imperator", "vic3"] {
+        let fp = format!("assets/tokens/{game}.txt");
+        let p = std::path::Path::new(&fp);
+        if !p.exists() {
+            std::fs::write(p, "").expect("to write file");
+        }
+    }
+}

src/json.rs

@@ -14,6 +14,11 @@ use std::{
 };
 use vic3save::Vic3File;
 
+use crate::tokens::{
+    ck3_tokens_resolver, eu4_tokens_resolver, hoi4_tokens_resolver, imperator_tokens_resolver,
+    vic3_tokens_resolver,
+};
+
 /// convert save and game files to json
 #[derive(FromArgs, PartialEq, Debug)]
 #[argh(subcommand, name = "json")]
@@ -79,21 +84,21 @@ impl JsonCommand {
                     file.melter()
                         .on_failed_resolve(strategy)
                         .verbatim(verbatim)
-                        .melt(&mut out, &eu4save::EnvTokens)?;
+                        .melt(&mut out, &eu4_tokens_resolver())?;
                     Eu4ParsedText::from_slice(out.get_ref().as_slice())?
                 } else {
                     Eu4ParsedText::from_slice(&data)?
                 };
 
                 text.reader().json().with_options(options).to_writer(writer)
             }
-            Some("ck3")  => {
+            Some("ck3") => {
                 let file = Ck3File::from_slice(&data)?;
                 let mut out = Cursor::new(Vec::new());
                 let text = if !matches!(file.encoding(), ck3save::Encoding::Text) {
                     file.melter()
                         .verbatim(true)
-                        .melt(&mut out, &ck3save::EnvTokens)?;
+                        .melt(&mut out, &ck3_tokens_resolver())?;
                     Ck3Text::from_slice(out.get_ref())?
                 } else {
                     Ck3Text::from_slice(&data)?
@@ -107,7 +112,7 @@ impl JsonCommand {
                     file.melter()
                         .on_failed_resolve(strategy)
                         .verbatim(verbatim)
-                        .melt(&mut out, &imperator_save::EnvTokens)?;
+                        .melt(&mut out, &imperator_tokens_resolver())?;
                     ImperatorText::from_slice(out.get_ref().as_slice())?
                 } else {
                     ImperatorText::from_slice(&data)?
@@ -121,7 +126,7 @@ impl JsonCommand {
                 let text = if !matches!(file.encoding(), hoi4save::Encoding::Plaintext) {
                     file.melter()
                         .verbatim(true)
-                        .melt(&mut out, &hoi4save::EnvTokens)?;
+                        .melt(&mut out, &hoi4_tokens_resolver())?;
                     Hoi4Text::from_slice(out.get_ref())?
                 } else {
                     Hoi4Text::from_slice(&data)?
@@ -133,7 +138,7 @@ impl JsonCommand {
                 let mut out = Cursor::new(Vec::new());
                 file.melter()
                     .verbatim(true)
-                    .melt(&mut out, &vic3save::EnvTokens)?;
+                    .melt(&mut out, &vic3_tokens_resolver())?;
                 let text = Hoi4Text::from_slice(out.get_ref())?;
                 text.reader().json().with_options(options).to_writer(writer)
             }

src/main.rs

@@ -1,6 +1,7 @@
 mod cli;
 mod json;
 mod melt;
+mod tokens;
 
 fn main() {
     std::process::exit(match cli::run() {

src/melt.rs

@@ -12,6 +12,11 @@ use std::{
     writeln,
 };
 
+use crate::tokens::{
+    ck3_tokens_resolver, eu4_tokens_resolver, hoi4_tokens_resolver, imperator_tokens_resolver,
+    vic3_tokens_resolver,
+};
+
 /// Melt a binary encoded file into the plaintext equivalent.
 #[derive(FromArgs, PartialEq, Debug)]
 #[argh(subcommand, name = "melt")]
@@ -112,7 +117,7 @@ impl Melter {
                     .melter()
                     .on_failed_resolve(self.options.resolve)
                     .verbatim(self.options.retain)
-                    .melt(writer, &eu4save::EnvTokens)?;
+                    .melt(writer, &eu4_tokens_resolver())?;
                 Ok(MeltedDocument::Eu4(out))
             }
             MelterKind::Ck3 => {
@@ -121,7 +126,7 @@ impl Melter {
                     .melter()
                     .on_failed_resolve(self.options.resolve)
                     .verbatim(self.options.retain)
-                    .melt(writer, &ck3save::EnvTokens)?;
+                    .melt(writer, &ck3_tokens_resolver())?;
                 Ok(MeltedDocument::Ck3(out))
             }
             MelterKind::Imperator => {
@@ -130,7 +135,7 @@ impl Melter {
                     .melter()
                     .on_failed_resolve(self.options.resolve)
                     .verbatim(self.options.retain)
-                    .melt(writer, &imperator_save::EnvTokens)?;
+                    .melt(writer, &imperator_tokens_resolver())?;
                 Ok(MeltedDocument::Imperator(out))
             }
             MelterKind::Vic3 => {
@@ -139,7 +144,7 @@ impl Melter {
                     .melter()
                     .on_failed_resolve(self.options.resolve)
                     .verbatim(self.options.retain)
-                    .melt(writer, &vic3save::EnvTokens)?;
+                    .melt(writer, &vic3_tokens_resolver())?;
                 Ok(MeltedDocument::Vic3(out))
             }
             MelterKind::Hoi4 => {
@@ -148,7 +153,7 @@ impl Melter {
                     .melter()
                     .on_failed_resolve(self.options.resolve)
                     .verbatim(self.options.retain)
-                    .melt(writer, &hoi4save::EnvTokens)?;
+                    .melt(writer, &hoi4_tokens_resolver())?;
                 Ok(MeltedDocument::Hoi4(out))
             }
         }

src/tokens.rs

@@ -0,0 +1,27 @@
+pub fn eu4_tokens_resolver() -> eu4save::BasicTokenResolver {
+    let data = include_bytes!("../assets/tokens/eu4.txt");
+    eu4save::BasicTokenResolver::from_text_lines(&data[..]).expect("embedded tokens invalid format")
+}
+
+pub fn ck3_tokens_resolver() -> ck3save::BasicTokenResolver {
+    let data = include_bytes!("../assets/tokens/ck3.txt");
+    ck3save::BasicTokenResolver::from_text_lines(&data[..]).expect("embedded tokens invalid format")
+}
+
+pub fn vic3_tokens_resolver() -> vic3save::BasicTokenResolver {
+    let data = include_bytes!("../assets/tokens/vic3.txt");
+    vic3save::BasicTokenResolver::from_text_lines(&data[..])
+        .expect("embedded tokens invalid format")
+}
+
+pub fn imperator_tokens_resolver() -> imperator_save::BasicTokenResolver {
+    let data = include_bytes!("../assets/tokens/imperator.txt");
+    imperator_save::BasicTokenResolver::from_text_lines(&data[..])
+        .expect("embedded tokens invalid format")
+}
+
+pub fn hoi4_tokens_resolver() -> hoi4save::BasicTokenResolver {
+    let data = include_bytes!("../assets/tokens/hoi4.txt");
+    hoi4save::BasicTokenResolver::from_text_lines(&data[..])
+        .expect("embedded tokens invalid format")
+}

tests/melt.rs

@@ -1,7 +1,7 @@
 mod utils;
 
 use assert_cmd::Command;
-use std::path::Path;
+use std::{collections::HashMap, path::Path};
 
 #[test]
 fn test_eu4_melt() {
@@ -16,7 +16,7 @@ fn test_eu4_melt() {
 
     let data = std::fs::read(&melted_path).unwrap();
     let file = eu4save::Eu4File::from_slice(&data).unwrap();
-    let _save = file.parse_save(&eu4save::EnvTokens).unwrap();
+    let _save = file.parse_save(&HashMap::<u16, &str>::new()).unwrap();
     assert_eq!(file.encoding(), eu4save::Encoding::Text)
 }
 
@@ -28,7 +28,7 @@ fn test_eu4_melt_stdout() {
 
     let out = assert.get_output();
     let file = eu4save::Eu4File::from_slice(&out.stdout).unwrap();
-    let _save = file.parse_save(&eu4save::EnvTokens).unwrap();
+    let _save = file.parse_save(&HashMap::<u16, &str>::new()).unwrap();
     assert_eq!(file.encoding(), eu4save::Encoding::Text)
 }
 
@@ -49,7 +49,7 @@ fn test_eu4_specify_format() {
 
     let out = assert.get_output();
     let file = eu4save::Eu4File::from_slice(&out.stdout).unwrap();
-    let _save = file.parse_save(&eu4save::EnvTokens).unwrap();
+    let _save = file.parse_save(&HashMap::<u16, &str>::new()).unwrap();
     assert_eq!(file.encoding(), eu4save::Encoding::Text)
 }
 
@@ -67,7 +67,7 @@ fn test_eu4_melt_to_out() {
 
     let data = std::fs::read(&output_path).unwrap();
     let file = eu4save::Eu4File::from_slice(&data).unwrap();
-    let _save = file.parse_save(&eu4save::EnvTokens).unwrap();
+    let _save = file.parse_save(&HashMap::<u16, &str>::new()).unwrap();
     assert_eq!(file.encoding(), eu4save::Encoding::Text)
 }
 
@@ -85,7 +85,7 @@ fn test_eu4_melt_stdin_to_stdout() {
 
     let out = assert.get_output();
     let file = eu4save::Eu4File::from_slice(&out.stdout).unwrap();
-    let _save = file.parse_save(&eu4save::EnvTokens).unwrap();
+    let _save = file.parse_save(&HashMap::<u16, &str>::new()).unwrap();
     assert_eq!(file.encoding(), eu4save::Encoding::Text)
 }
 
@@ -102,7 +102,7 @@ fn test_eu4_melt_retain() {
 
     let out = assert.get_output();
     let file = eu4save::Eu4File::from_slice(&out.stdout).unwrap();
-    let _save = file.parse_save(&eu4save::EnvTokens).unwrap();
+    let _save = file.parse_save(&HashMap::<u16, &str>::new()).unwrap();
     assert_eq!(file.encoding(), eu4save::Encoding::Text)
 }
 

tests/utils.rs

@@ -1,10 +1,17 @@
-use std::path::{Path, PathBuf};
+use std::{path::{Path, PathBuf}, sync::Mutex};
+
+static DOWNLOADER: Mutex<()> = Mutex::new(());
 
 /// Request data from s3 and cache it locally
 pub fn request<S: AsRef<str>>(bucket_name: &str, input: S) -> PathBuf {
     let reffed = input.as_ref();
     let cache = Path::new("assets").join("saves").join(reffed);
     if !cache.exists() {
+        let _guard = DOWNLOADER.lock().unwrap();
+        if cache.exists() {
+            return cache;
+        }
+
         let url = format!(
             "https://{}.s3.us-west-002.backblazeb2.com/{}",
             bucket_name, reffed