Speedup codegen by caching tools/codegen compilation (#554)

.github/workflows/ci.yml

@@ -83,6 +83,8 @@ jobs:
       - uses: ./
         with:
           tool: ${{ steps.tool-list.outputs.tool }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
       - name: Test bash
         run: just --version && shfmt --version && protoc --version
@@ -151,6 +153,8 @@ jobs:
       - uses: ./
         with:
           tool: ${{ steps.tool-list.outputs.tool }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   manifest:
     runs-on: ubuntu-latest
@@ -161,7 +165,12 @@ jobs:
     steps:
       - uses: taiki-e/checkout-action@v1
       - name: Install Rust
-        run: rustup toolchain add nightly --no-self-update && rustup default nightly
+        run: rustup update stable --no-self-update
+      - name: Generate Cargo.lock
+        run: cargo update
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-all-crates: 'true'
       - run: tools/manifest.sh
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

tools/codegen/src/main.rs

@@ -7,7 +7,7 @@ use std::{
     ffi::OsStr,
     io::Read,
     path::Path,
-    sync::{LazyLock, RwLock},
+    sync::{OnceLock, RwLock},
     time::Duration,
 };
 
@@ -649,38 +649,43 @@ struct GitHubTokens {
     other: RwLock<Option<String>>,
 }
 impl GitHubTokens {
-    fn get(&self, url: &str) -> Option<String> {
+    // TODO: Use std::sync::LazyLock once 1.80 is released
-        if url.starts_with("https://raw.githubusercontent.com/") {
+    fn get_github_tokens() -> &'static GitHubTokens {
-            self.raw.read().unwrap().clone()
+        static GITHUB_TOKENS: OnceLock<GitHubTokens> = OnceLock::new();
-        } else if url.starts_with("https://api.github.com/") {
+        GITHUB_TOKENS.get_or_init(|| {
-            self.api.read().unwrap().clone()
-        } else if url.starts_with("https://github.com/") {
-            self.other.read().unwrap().clone()
-        } else {
-            None
-        }
-    }
-    fn clear(&self, url: &str) {
-        if url.starts_with("https://raw.githubusercontent.com/") {
-            *self.raw.write().unwrap() = None;
-        } else if url.starts_with("https://api.github.com/") {
-            *self.api.write().unwrap() = None;
-        } else if url.starts_with("https://github.com/") {
-            *self.other.write().unwrap() = None;
-        }
-    }
-}
-static GITHUB_TOKENS: LazyLock<GitHubTokens> = LazyLock::new(|| {
             let token = env::var("GITHUB_TOKEN").ok().filter(|v| !v.is_empty());
             GitHubTokens {
                 raw: RwLock::new(token.clone()),
                 api: RwLock::new(token.clone()),
                 other: RwLock::new(token),
             }
-});
+        })
+    }
+
+    fn get(url: &str) -> Option<String> {
+        if url.starts_with("https://raw.githubusercontent.com/") {
+            Self::get_github_tokens().raw.read().unwrap().clone()
+        } else if url.starts_with("https://api.github.com/") {
+            Self::get_github_tokens().api.read().unwrap().clone()
+        } else if url.starts_with("https://github.com/") {
+            Self::get_github_tokens().other.read().unwrap().clone()
+        } else {
+            None
+        }
+    }
+    fn clear(url: &str) {
+        if url.starts_with("https://raw.githubusercontent.com/") {
+            *Self::get_github_tokens().raw.write().unwrap() = None;
+        } else if url.starts_with("https://api.github.com/") {
+            *Self::get_github_tokens().api.write().unwrap() = None;
+        } else if url.starts_with("https://github.com/") {
+            *Self::get_github_tokens().other.write().unwrap() = None;
+        }
+    }
+}
 
 fn download(url: &str) -> Result<ureq::Response> {
-    let mut token = GITHUB_TOKENS.get(url);
+    let mut token = GitHubTokens::get(url);
     let mut retry = 0;
     let mut retry_time = 0;
     let mut max_retry = 6;
@@ -702,7 +707,7 @@ fn download(url: &str) -> Result<ureq::Response> {
             retry_time = 0;
             token = None;
             // rate limit
-            GITHUB_TOKENS.clear(url);
+            GitHubTokens::clear(url);
         }
         retry += 1;
         if retry > max_retry {
@@ -716,7 +721,7 @@ fn download(url: &str) -> Result<ureq::Response> {
 
 fn github_head(url: &str) -> Result<()> {
     eprintln!("fetching head of {url} ..");
-    let mut token = GITHUB_TOKENS.get(url);
+    let mut token = GitHubTokens::get(url);
     let mut retry = 0;
     let mut retry_time = 0;
     let mut max_retry = 2;
@@ -739,7 +744,7 @@ fn github_head(url: &str) -> Result<()> {
         if token.is_some() && retry == max_retry / 2 {
             retry_time = 0;
             token = None;
-            GITHUB_TOKENS.clear(url);
+            GitHubTokens::clear(url);
         }
         retry += 1;
         if retry > max_retry {

tools/manifest.sh

@@ -10,11 +10,11 @@ cd "$(dirname "$0")"/..
 #    ./tools/manifest.sh [PACKAGE [VERSION_REQ]]
 
 if [[ $# -gt 0 ]]; then
-    cargo +nightly run --manifest-path tools/codegen/Cargo.toml --release -- "$@"
+    cargo run --manifest-path tools/codegen/Cargo.toml --release -- "$@"
     exit 0
 fi
 
 for manifest in tools/codegen/base/*.json; do
     package=$(basename "${manifest%.*}")
-    cargo +nightly run --manifest-path tools/codegen/Cargo.toml --release -- "${package}" latest
+    cargo run --manifest-path tools/codegen/Cargo.toml --release -- "${package}" latest
 done