tools: Update scripts

This commit is contained in:
Taiki Endo
2025-12-04 02:45:49 +09:00
parent 8aeb767d80
commit c8c9b5bb95
4 changed files with 55 additions and 49 deletions

View File

@@ -1,5 +1,5 @@
// This file is @generated by tidy.sh.
// It is not intended for manual editing.
# This file is @generated by tidy.sh.
# It is not intended for manual editing.
flate
minisign

4
.github/zizmor.yml vendored
View File

@@ -2,7 +2,7 @@
# https://docs.zizmor.sh/configuration/
rules:
dependabot-cooldown: { disable: true } # Useless unless unpinned-uses is enabled.
dependabot-cooldown: { disable: true } # Useless unless hash-pin is forced by unpinned-uses.
ref-confusion: { disable: true } # TODO: Old GHA didn't work without this pattern in some cases, but does it seem to be fixed?
secrets-inherit: { disable: true }
unpinned-uses:
@@ -12,5 +12,5 @@ rules:
'*': ref-pin
obfuscation:
ignore:
# We use `shell: cmd` to test compatibility
# We use `shell: cmd` to test compatibility.
- ci.yml

View File

@@ -191,7 +191,7 @@ cp -- ./manifests/* "${schema_workspace}"
# Stage changes
git add .
# Detect changes, then commit and push if changes exist
if [[ "$(git status --porcelain=v1 | wc -l)" != "0" ]]; then
if [[ "$(git status --porcelain=v1 | LC_ALL=C wc -l)" != "0" ]]; then
git commit -m 'Update manifest schema'
retry git push origin HEAD
fi

View File

@@ -118,11 +118,11 @@ check_alt() {
check_hidden() {
local res
for file in "$@"; do
check_alt ".${file}" "${file}" "$(comm -23 <(ls_files "*${file}") <(ls_files "*.${file}"))"
check_alt ".${file}" "${file}" "$(LC_ALL=C comm -23 <(ls_files "*${file}") <(ls_files "*.${file}"))"
done
}
sed_rhs_escape() {
sed 's/\\/\\\\/g; s/\&/\\\&/g; s/\//\\\//g' <<<"$1"
sed -E 's/\\/\\\\/g; s/\&/\\\&/g; s/\//\\\//g' <<<"$1"
}
if [[ $# -gt 0 ]]; then
@@ -137,12 +137,8 @@ py_suffix=''
if type -P python3 >/dev/null; then
py_suffix=3
fi
yq() {
pipx run yq "$@"
}
tomlq() {
pipx run --spec yq tomlq "$@"
}
yq() { pipx run yq "$@"; }
tomlq() { pipx run --spec yq tomlq "$@"; }
case "$(uname -s)" in
Linux)
if [[ "$(uname -o)" == 'Android' ]]; then
@@ -167,10 +163,11 @@ case "$(uname -s)" in
if [[ "${PATH}" != *'/usr/xpg4/bin'* ]]; then
export PATH="/usr/xpg4/bin:${PATH}"
fi
# GNU/BSD grep/sed is required to run some checks, but most checks are okay with other POSIX grep/sed.
# GNU/BSD sed is required.
# GNU/BSD grep is required by some checks, but most checks are okay with other POSIX grep.
# Solaris /usr/xpg4/bin/grep has -q, -E, -F, but no -o (non-POSIX).
# Solaris /usr/xpg4/bin/sed has no -E (POSIX.1-2024) yet.
for tool in sed grep; do
for tool in 'grep' 'sed'; do
if type -P "g${tool}" >/dev/null; then
eval "${tool}() { g${tool} \"\$@\"; }"
fi
@@ -189,12 +186,8 @@ case "$(uname -s)" in
else
jq() { command jq "$@" | tr -d '\r'; }
fi
yq() {
pipx run yq "$@" | tr -d '\r'
}
tomlq() {
pipx run --spec yq tomlq "$@" | tr -d '\r'
}
yq() { pipx run yq "$@" | tr -d '\r'; }
tomlq() { pipx run --spec yq tomlq "$@" | tr -d '\r'; }
fi
fi
;;
@@ -203,25 +196,34 @@ esac
check_install git
exclude_from_ls_files=()
# - `find` lists symlinks. `! ( -name <dir> -prune )` (.i.e., ignore <dir>) are manually listed from .gitignore.
# - `git submodule status` lists submodules. Use sed to remove the first character indicates status ( |+|-).
# - `find` lists symlinks. `! ( -name <dir> -prune )` means recursively ignore <dir>. `cut` removes the leading `./`.
# This can be replaced with `fd -H -t l`.
# - `git submodule status` lists submodules. The first `cut` removes the first character indicates status ( |+|-).
# - `git ls-files --deleted` lists removed files.
while IFS=$'\n' read -r line; do exclude_from_ls_files+=("${line}"); done < <({
find . \! \( -name .git -prune \) \! \( -name target -prune \) \! \( -name tmp -prune \) -type l | cut -c3-
git submodule status | sed 's/^.//' | cut -d' ' -f2
find_prune=(\! \( -name .git -prune \))
while IFS= read -r; do
find_prune+=(\! \( -name "${REPLY}" -prune \))
done < <(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' .gitignore)
while IFS=$'\n' read -r; do
exclude_from_ls_files+=("${REPLY}")
done < <({
find . "${find_prune[@]}" -type l | cut -c3-
git submodule status | cut -c2- | cut -d' ' -f2
git ls-files --deleted
} | LC_ALL=C sort -u)
exclude_from_ls_files_no_symlink=()
while IFS=$'\n' read -r line; do exclude_from_ls_files_no_symlink+=("${line}"); done < <({
git submodule status | sed 's/^.//' | cut -d' ' -f2
while IFS=$'\n' read -r; do
exclude_from_ls_files_no_symlink+=("${REPLY}")
done < <({
git submodule status | cut -c2- | cut -d' ' -f2
git ls-files --deleted
} | LC_ALL=C sort -u)
ls_files() {
if [[ "${1:-}" == '--include-symlink' ]]; then
shift
comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files_no_symlink[@]+"${exclude_from_ls_files_no_symlink[@]}"})
LC_ALL=C comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files_no_symlink[@]+"${exclude_from_ls_files_no_symlink[@]}"})
else
comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files[@]+"${exclude_from_ls_files[@]}"})
LC_ALL=C comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files[@]+"${exclude_from_ls_files[@]}"})
fi
}
@@ -445,7 +447,7 @@ if [[ -n "$(ls_files '*.rs')" ]]; then
new+="${line}"$'\a'
done < <(tr '\n' '\a' <"${markdown}" | grep -Eo '<!-- tidy:sync-markdown-to-rustdoc:start[^ ]* -->.*<!-- tidy:sync-markdown-to-rustdoc:end -->')
new+='<!-- tidy:sync-markdown-to-rustdoc:end -->'
new=$(tr '\n' '\a' <"${lib}" | sed "s/<!-- tidy:sync-markdown-to-rustdoc:start[^ ]* -->.*<!-- tidy:sync-markdown-to-rustdoc:end -->/$(sed_rhs_escape "${new}")/" | tr '\a' '\n')
new=$(tr '\n' '\a' <"${lib}" | sed -E "s/<!-- tidy:sync-markdown-to-rustdoc:start[^ ]* -->.*<!-- tidy:sync-markdown-to-rustdoc:end -->/$(sed_rhs_escape "${new}")/" | tr '\a' '\n')
printf '%s\n' "${new}" >|"${lib}"
check_diff "${lib}"
done
@@ -677,7 +679,7 @@ elif check_install shellcheck; then
# Others: false negative
trap -- 'rm -- ./tools/.tidy-tmp; printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT
printf '%s\n' "${text}" >|./tools/.tidy-tmp
if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then
if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed -E "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then
error "check failed; please resolve the above shellcheck error(s)"
fi
rm -- ./tools/.tidy-tmp
@@ -822,7 +824,7 @@ EOF
# Others: false negative
trap -- 'rm -- ./tools/.tidy-tmp; printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT
printf '%s\n' "${text}" >|./tools/.tidy-tmp
if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then
if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed -E "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then
error "check failed; please resolve the above shellcheck error(s)"
fi
rm -- ./tools/.tidy-tmp
@@ -833,7 +835,8 @@ EOF
# The top-level permissions must be weak as they are referenced by all jobs.
permissions=$(jq -c '.permissions' <<<"${workflow}")
case "${permissions}" in
'{"contents":"read"}' | '{"contents":"none"}') ;;
# `permissions: {}` means "all none": https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defining-access-for-the-github_token-scopes
'{"contents":"read"}' | '{}') ;;
null) error "${workflow_path}: top level permissions not found; it must be 'contents: read' or weaker permissions" ;;
*) error "${workflow_path}: only 'contents: read' and weaker permissions are allowed at top level, but found '${permissions}'; if you want to use stronger permissions, please set job-level permissions" ;;
esac
@@ -911,11 +914,13 @@ if [[ -e .github/dependabot.yml ]]; then
zizmor_targets+=(.github/dependabot.yml)
fi
if [[ ${#zizmor_targets[@]} -gt 0 ]]; then
if check_install zizmor; then
if [[ "${ostype}" =~ ^(netbsd|openbsd|dragonfly|illumos|solaris)$ ]] && [[ -n "${CI:-}" ]] && ! type -P zizmor >/dev/null; then
warn "this check is skipped on NetBSD/OpenBSD/Dragonfly/illumos/Solaris due to installing zizmor is hard on these platform"
elif check_install zizmor; then
IFS=' '
info "running \`zizmor ${zizmor_targets[*]}\`"
info "running \`zizmor -q ${zizmor_targets[*]}\`"
IFS=$'\n\t'
zizmor "${zizmor_targets[@]}"
zizmor -q "${zizmor_targets[@]}"
fi
fi
printf '\n'
@@ -926,7 +931,7 @@ check_alt '.sh extension' '*.bash extension' "$(ls_files '*.bash')"
if [[ -f tools/.tidy-check-license-headers ]]; then
info "checking license headers (experimental)"
failed_files=''
for p in $(comm -12 <(eval $(<tools/.tidy-check-license-headers) | LC_ALL=C sort) <(ls_files | LC_ALL=C sort)); do
for p in $(LC_ALL=C comm -12 <(eval $(<tools/.tidy-check-license-headers) | LC_ALL=C sort) <(ls_files | LC_ALL=C sort)); do
case "${p##*/}" in
*.stderr | *.expanded.rs) continue ;; # generated files
*.json) continue ;; # no comment support
@@ -998,8 +1003,8 @@ if [[ -f .cspell.json ]]; then
printf '%s\n' "${config_old}" >|.cspell.json
trap -- 'printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT
cat >|.github/.cspell/rust-dependencies.txt <<EOF
// This file is @generated by ${0##*/}.
// It is not intended for manual editing.
# This file is @generated by ${0##*/}.
# It is not intended for manual editing.
EOF
if [[ -n "${dependencies_words}" ]]; then
LC_ALL=C sort -f >>.github/.cspell/rust-dependencies.txt <<<"${dependencies_words}"$'\n'
@@ -1019,7 +1024,7 @@ EOF
if ! ls_files | npx -y cspell stdin --no-progress --no-summary --show-context; then
error "spellcheck failed: please fix uses of below words in file names or add to ${project_dictionary} if correct"
printf '=======================================\n'
{ ls_files | npx -y cspell stdin --no-progress --no-summary --words-only || true; } | sed "s/'s$//g" | LC_ALL=C sort -f -u
{ ls_files | npx -y cspell stdin --no-progress --no-summary --words-only || true; } | sed -E "s/'s$//g" | LC_ALL=C sort -f -u
printf '=======================================\n\n'
fi
# Check file contains.
@@ -1027,7 +1032,7 @@ EOF
if ! ls_files | npx -y cspell --file-list stdin --no-progress --no-summary; then
error "spellcheck failed: please fix uses of below words or add to ${project_dictionary} if correct"
printf '=======================================\n'
{ ls_files | npx -y cspell --file-list stdin --no-progress --no-summary --words-only || true; } | sed "s/'s$//g" | LC_ALL=C sort -f -u
{ ls_files | npx -y cspell --file-list stdin --no-progress --no-summary --words-only || true; } | sed -E "s/'s$//g" | LC_ALL=C sort -f -u
printf '=======================================\n\n'
fi
@@ -1038,8 +1043,8 @@ EOF
fi
case "${ostype}" in
# NetBSD uniq doesn't support -i flag.
netbsd) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | tr '[:upper:]' '[:lower:]' | LC_ALL=C uniq -d) ;;
*) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | LC_ALL=C uniq -d -i) ;;
netbsd) dup=$(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | tr '[:upper:]' '[:lower:]' | LC_ALL=C uniq -d) ;;
*) dup=$(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | LC_ALL=C uniq -d -i) ;;
esac
if [[ -n "${dup}" ]]; then
error "duplicated words in dictionaries; please remove the following words from ${project_dictionary}"
@@ -1050,13 +1055,14 @@ EOF
# Make sure the project-specific dictionary does not contain unused words.
if [[ -n "${REMOVE_UNUSED_WORDS:-}" ]]; then
grep_args=()
for word in $(grep -Ev '^//' "${project_dictionary}" || true); do
while IFS= read -r word; do
if ! grep -Eqi "^${word}$" <<<"${all_words}"; then
grep_args+=(-e "^${word}$")
grep_args+=(-e "^[ \t]*${word}[ \t]*(#.*|$)")
fi
done
done < <(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}")
if [[ ${#grep_args[@]} -gt 0 ]]; then
info "removing unused words from ${project_dictionary}"
info "please commit changes made by the removal above"
res=$(grep -Ev "${grep_args[@]}" "${project_dictionary}" || true)
if [[ -n "${res}" ]]; then
printf '%s\n' "${res}" >|"${project_dictionary}"
@@ -1066,11 +1072,11 @@ EOF
fi
else
unused=''
for word in $(grep -Ev '^//' "${project_dictionary}" || true); do
while IFS= read -r word; do
if ! grep -Eqi "^${word}$" <<<"${all_words}"; then
unused+="${word}"$'\n'
fi
done
done < <(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}")
if [[ -n "${unused}" ]]; then
error "unused words in dictionaries; please remove the following words from ${project_dictionary} or run ${0##*/} locally"
print_fenced "${unused}"