Translation cleanup (#33738)
This commit is contained in:
Kevin Heis
@@ -1 +0,0 @@
|
|||||||
ALLOW_TRANSLATION_COMMITS=
|
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
import fs from 'fs'
|
|
||||||
import github from '@actions/github'
|
|
||||||
|
|
||||||
const OPTIONS = Object.fromEntries(
|
|
||||||
['BASE', 'BODY_FILE', 'GITHUB_TOKEN', 'HEAD', 'LANGUAGE', 'TITLE', 'GITHUB_REPOSITORY'].map(
|
|
||||||
(envVarName) => {
|
|
||||||
const envVarValue = process.env[envVarName]
|
|
||||||
if (!envVarValue) {
|
|
||||||
throw new Error(`You must supply a ${envVarName} environment variable`)
|
|
||||||
}
|
|
||||||
return [envVarName, envVarValue]
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!process.env.GITHUB_REPOSITORY) {
|
|
||||||
throw new Error('GITHUB_REPOSITORY environment variable not set')
|
|
||||||
}
|
|
||||||
|
|
||||||
const RETRY_STATUSES = [
|
|
||||||
422, // Retry the operation if the PR already exists
|
|
||||||
502, // Retry the operation if the API responds with a `502 Bad Gateway` error.
|
|
||||||
]
|
|
||||||
const RETRY_ATTEMPTS = 3
|
|
||||||
const {
|
|
||||||
// One of the default environment variables provided by Actions.
|
|
||||||
GITHUB_REPOSITORY,
|
|
||||||
|
|
||||||
// These are passed in from the step in the workflow file.
|
|
||||||
TITLE,
|
|
||||||
BASE,
|
|
||||||
HEAD,
|
|
||||||
LANGUAGE,
|
|
||||||
BODY_FILE,
|
|
||||||
GITHUB_TOKEN,
|
|
||||||
} = OPTIONS
|
|
||||||
const [OWNER, REPO] = GITHUB_REPOSITORY.split('/')
|
|
||||||
|
|
||||||
const octokit = github.getOctokit(GITHUB_TOKEN)
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {object} config Configuration options for finding the PR.
|
|
||||||
* @returns {Promise<number | undefined>} The PR number.
|
|
||||||
*/
|
|
||||||
async function findPullRequestNumber(config) {
|
|
||||||
// Get a list of PRs and see if one already exists.
|
|
||||||
const { data: listOfPullRequests } = await octokit.rest.pulls.list({
|
|
||||||
owner: config.owner,
|
|
||||||
repo: config.repo,
|
|
||||||
head: `${config.owner}:${config.head}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
return listOfPullRequests[0]?.number
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* When this file was first created, we only introduced support for creating a pull request for some translation batch.
|
|
||||||
* However, some of our first workflow runs failed during the pull request creation due to a timeout error.
|
|
||||||
* There have been cases where, despite the timeout error, the pull request gets created _anyway_.
|
|
||||||
* To accommodate this reality, we created this function to look for an existing pull request before a new one is created.
|
|
||||||
* Although the "find" check is redundant in the first "cycle", it's designed this way to recursively call the function again via its retry mechanism should that be necessary.
|
|
||||||
*
|
|
||||||
* @param {object} config Configuration options for creating the pull request.
|
|
||||||
* @returns {Promise<number>} The PR number.
|
|
||||||
*/
|
|
||||||
async function findOrCreatePullRequest(config) {
|
|
||||||
const found = await findPullRequestNumber(config)
|
|
||||||
|
|
||||||
if (found) {
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const { data: pullRequest } = await octokit.rest.pulls.create({
|
|
||||||
owner: config.owner,
|
|
||||||
repo: config.repo,
|
|
||||||
base: config.base,
|
|
||||||
head: config.head,
|
|
||||||
title: config.title,
|
|
||||||
body: config.body,
|
|
||||||
draft: false,
|
|
||||||
})
|
|
||||||
|
|
||||||
return pullRequest.number
|
|
||||||
} catch (error) {
|
|
||||||
if (!error.response || !config.retryCount) {
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!config.retryStatuses.includes(error.response.status)) {
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
|
|
||||||
console.error(`Error creating pull request: ${error.message}`)
|
|
||||||
console.warn(`Retrying in 5 seconds...`)
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 5000))
|
|
||||||
|
|
||||||
config.retryCount -= 1
|
|
||||||
|
|
||||||
return findOrCreatePullRequest(config)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {object} config Configuration options for labeling the PR
|
|
||||||
* @returns {Promise<undefined>}
|
|
||||||
*/
|
|
||||||
async function labelPullRequest(config) {
|
|
||||||
await octokit.rest.issues.update({
|
|
||||||
owner: config.owner,
|
|
||||||
repo: config.repo,
|
|
||||||
issue_number: config.issue_number,
|
|
||||||
labels: config.labels,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const options = {
|
|
||||||
title: TITLE,
|
|
||||||
base: BASE,
|
|
||||||
head: HEAD,
|
|
||||||
body: fs.readFileSync(BODY_FILE, 'utf8'),
|
|
||||||
labels: ['translation-batch', `translation-batch-${LANGUAGE}`],
|
|
||||||
owner: OWNER,
|
|
||||||
repo: REPO,
|
|
||||||
retryStatuses: RETRY_STATUSES,
|
|
||||||
retryCount: RETRY_ATTEMPTS,
|
|
||||||
}
|
|
||||||
|
|
||||||
options.issue_number = await findOrCreatePullRequest(options)
|
|
||||||
const pr = `${GITHUB_REPOSITORY}#${options.issue_number}`
|
|
||||||
console.log(`Created PR ${pr}`)
|
|
||||||
|
|
||||||
// metadata parameters aren't currently available in `github.rest.pulls.create`,
|
|
||||||
// but they are in `github.rest.issues.update`.
|
|
||||||
await labelPullRequest(options)
|
|
||||||
console.log(`Updated ${pr} with these labels: ${options.labels.join(', ')}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
main()
|
|
||||||
@@ -1,193 +0,0 @@
|
|||||||
name: Create translation Batch Pull Request (Microsoft)
|
|
||||||
|
|
||||||
# **What it does**:
|
|
||||||
# - Creates one pull request per language after running a series of automated checks,
|
|
||||||
# removing translations that are broken in any known way
|
|
||||||
# **Why we have it**:
|
|
||||||
# - To deploy translations
|
|
||||||
# **Who does it impact**: It automates what would otherwise be manual work,
|
|
||||||
# helping docs engineering focus on higher value work
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
schedule:
|
|
||||||
- cron: '20 16 * * *' # Run every day at 16:20 UTC / 8:20 PST
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
create-translation-batch:
|
|
||||||
name: Create translation batch
|
|
||||||
if: github.repository == 'github/docs-internal'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
# A sync's average run time is ~3.2 hours.
|
|
||||||
# This sets a maximum execution time of 300 minutes (5 hours) to prevent the workflow from running longer than necessary.
|
|
||||||
timeout-minutes: 300
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
max-parallel: 1
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- language: es
|
|
||||||
language_dir: translations/es-ES
|
|
||||||
language_repo: github/docs-internal.es-es
|
|
||||||
|
|
||||||
- language: ja
|
|
||||||
language_dir: translations/ja-JP
|
|
||||||
language_repo: github/docs-internal.ja-jp
|
|
||||||
|
|
||||||
- language: pt
|
|
||||||
language_dir: translations/pt-BR
|
|
||||||
language_repo: github/docs-internal.pt-br
|
|
||||||
|
|
||||||
- language: zh
|
|
||||||
language_dir: translations/zh-CN
|
|
||||||
language_repo: github/docs-internal.zh-cn
|
|
||||||
|
|
||||||
# We'll be ready to add the following languages in a future effort.
|
|
||||||
|
|
||||||
- language: ru
|
|
||||||
language_dir: translations/ru-RU
|
|
||||||
language_repo: github/docs-internal.ru-ru
|
|
||||||
|
|
||||||
- language: ko
|
|
||||||
language_dir: translations/ko-KR
|
|
||||||
language_repo: github/docs-internal.ko-kr
|
|
||||||
|
|
||||||
- language: fr
|
|
||||||
language_dir: translations/fr-FR
|
|
||||||
language_repo: github/docs-internal.fr-fr
|
|
||||||
|
|
||||||
- language: de
|
|
||||||
language_dir: translations/de-DE
|
|
||||||
language_repo: github/docs-internal.de-de
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Set branch name
|
|
||||||
id: set-branch
|
|
||||||
run: |
|
|
||||||
echo "BRANCH_NAME=msft-translation-batch-${{ matrix.language }}-$(date +%Y-%m-%d__%H-%M)" >> $GITHUB_OUTPUT
|
|
||||||
- run: git config --global user.name "docubot"
|
|
||||||
- run: git config --global user.email "67483024+docubot@users.noreply.github.com"
|
|
||||||
|
|
||||||
- name: Checkout the docs-internal repo
|
|
||||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
lfs: true
|
|
||||||
|
|
||||||
- name: Create a branch for the current language
|
|
||||||
run: git checkout -b ${{ steps.set-branch.outputs.BRANCH_NAME }}
|
|
||||||
|
|
||||||
- name: Remove unwanted git hooks
|
|
||||||
run: rm .git/hooks/post-checkout
|
|
||||||
|
|
||||||
- name: Remove all language translations
|
|
||||||
run: |
|
|
||||||
git rm -rf --quiet ${{ matrix.language_dir }}/content
|
|
||||||
git rm -rf --quiet ${{ matrix.language_dir }}/data
|
|
||||||
|
|
||||||
- name: Checkout the language-specific repo
|
|
||||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
|
|
||||||
with:
|
|
||||||
repository: ${{ matrix.language_repo }}
|
|
||||||
token: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }}
|
|
||||||
path: ${{ matrix.language_dir }}
|
|
||||||
|
|
||||||
- name: Remove .git from the language-specific repo
|
|
||||||
run: rm -rf ${{ matrix.language_dir }}/.git
|
|
||||||
|
|
||||||
- name: Commit translated files
|
|
||||||
run: |
|
|
||||||
git add ${{ matrix.language_dir }}
|
|
||||||
git commit -m "Add translations" || echo "Nothing to commit"
|
|
||||||
|
|
||||||
- name: 'Setup node'
|
|
||||||
uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516
|
|
||||||
with:
|
|
||||||
node-version: '16.17.0'
|
|
||||||
|
|
||||||
- run: npm ci
|
|
||||||
|
|
||||||
- name: Homogenize frontmatter
|
|
||||||
run: |
|
|
||||||
node script/i18n/homogenize-frontmatter.js
|
|
||||||
git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit"
|
|
||||||
|
|
||||||
- name: Fix translation errors
|
|
||||||
run: |
|
|
||||||
node script/i18n/fix-translation-errors.js
|
|
||||||
git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit"
|
|
||||||
|
|
||||||
- name: Check rendering
|
|
||||||
run: |
|
|
||||||
node script/i18n/lint-translation-files.js --check rendering | tee -a /tmp/batch.log | cat
|
|
||||||
git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit"
|
|
||||||
|
|
||||||
- name: Reset files with broken liquid tags
|
|
||||||
run: |
|
|
||||||
node script/i18n/msft-reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }} | tee -a /tmp/batch.log | cat
|
|
||||||
git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/msft-reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }}" || echo "Nothing to commit"
|
|
||||||
|
|
||||||
- name: Check in CSV report
|
|
||||||
run: |
|
|
||||||
mkdir -p translations/log
|
|
||||||
csvFile=translations/log/msft-${{ matrix.language }}-resets.csv
|
|
||||||
script/i18n/msft-report-reset-files.js --report-type=csv --language=${{ matrix.language }} --log-file=/tmp/batch.log > $csvFile
|
|
||||||
git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit"
|
|
||||||
|
|
||||||
- name: Write the reported files that were reset to /tmp/pr-body.txt
|
|
||||||
run: script/i18n/msft-report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language }} --log-file=/tmp/batch.log --csv-path=${{ steps.set-branch.outputs.BRANCH_NAME }}/translations/log/msft-${{ matrix.language }}-resets.csv > /tmp/pr-body.txt
|
|
||||||
|
|
||||||
- name: Push filtered translations
|
|
||||||
run: git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }}
|
|
||||||
|
|
||||||
- name: Close existing stale batches
|
|
||||||
uses: lee-dohm/close-matching-issues@e9e43aad2fa6f06a058cedfd8fb975fd93b56d8f
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
|
|
||||||
query: 'type:pr label:translation-batch-${{ matrix.language }}'
|
|
||||||
|
|
||||||
- name: Create translation batch pull request
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }}
|
|
||||||
TITLE: 'New translation batch for ${{ matrix.language }}'
|
|
||||||
BASE: 'main'
|
|
||||||
HEAD: ${{ steps.set-branch.outputs.BRANCH_NAME }}
|
|
||||||
LANGUAGE: ${{ matrix.language }}
|
|
||||||
BODY_FILE: '/tmp/pr-body.txt'
|
|
||||||
run: .github/actions-scripts/msft-create-translation-batch-pr.js
|
|
||||||
|
|
||||||
- name: Approve PR
|
|
||||||
if: github.ref_name == 'main'
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
|
|
||||||
run: gh pr review --approve || echo "Nothing to approve"
|
|
||||||
|
|
||||||
- name: Set auto-merge
|
|
||||||
if: github.ref_name == 'main'
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
|
|
||||||
run: gh pr merge ${{ steps.set-branch.outputs.BRANCH_NAME }} --auto --squash || echo "Nothing to merge"
|
|
||||||
|
|
||||||
# When the maximum execution time is reached for this job, Actions cancels the workflow run.
|
|
||||||
# This emits a notification for the first responder to triage.
|
|
||||||
- name: Send Slack notification if workflow is cancelled
|
|
||||||
uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
|
|
||||||
if: cancelled()
|
|
||||||
with:
|
|
||||||
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
|
|
||||||
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}🎉
|
|
||||||
color: failure
|
|
||||||
text: 'The new translation batch for ${{ matrix.language }} was cancelled.'
|
|
||||||
|
|
||||||
# Emit a notification for the first responder to triage if the workflow failed.
|
|
||||||
- name: Send Slack notification if workflow failed
|
|
||||||
uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
|
|
||||||
if: failure()
|
|
||||||
with:
|
|
||||||
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
|
|
||||||
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
|
|
||||||
color: failure
|
|
||||||
text: 'The new translation batch for ${{ matrix.language }} failed.'
|
|
||||||
@@ -20,7 +20,6 @@ on:
|
|||||||
- 'lib/webhooks/**'
|
- 'lib/webhooks/**'
|
||||||
- 'package*.json'
|
- 'package*.json'
|
||||||
- 'script/**'
|
- 'script/**'
|
||||||
- 'translations/**'
|
|
||||||
- 'content/actions/deployment/security-hardening-your-deployments/**'
|
- 'content/actions/deployment/security-hardening-your-deployments/**'
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
@@ -49,8 +48,6 @@ jobs:
|
|||||||
|
|
||||||
# Returns list of changed files matching each filter
|
# Returns list of changed files matching each filter
|
||||||
filters: |
|
filters: |
|
||||||
translation:
|
|
||||||
- 'translations/**'
|
|
||||||
openapi:
|
openapi:
|
||||||
- 'lib/rest/static/**'
|
- 'lib/rest/static/**'
|
||||||
notAllowed:
|
notAllowed:
|
||||||
@@ -67,7 +64,6 @@ jobs:
|
|||||||
- 'lib/webhooks/**'
|
- 'lib/webhooks/**'
|
||||||
- 'package*.json'
|
- 'package*.json'
|
||||||
- 'scripts/**'
|
- 'scripts/**'
|
||||||
- 'translations/**'
|
|
||||||
- 'content/actions/deployment/security-hardening-your-deployments/**'
|
- 'content/actions/deployment/security-hardening-your-deployments/**'
|
||||||
|
|
||||||
# When there are changes to files we can't accept, leave a comment
|
# When there are changes to files we can't accept, leave a comment
|
||||||
@@ -91,7 +87,6 @@ jobs:
|
|||||||
'lib/webhooks/**',
|
'lib/webhooks/**',
|
||||||
'package*.json',
|
'package*.json',
|
||||||
'scripts/**',
|
'scripts/**',
|
||||||
'translations/**',
|
|
||||||
'content/actions/deployment/security-hardening-your-deployments/**',
|
'content/actions/deployment/security-hardening-your-deployments/**',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
/translations/
|
|
||||||
includes/
|
includes/
|
||||||
data/release-notes/
|
data/release-notes/
|
||||||
script/bookmarklets/
|
script/bookmarklets/
|
||||||
|
|||||||
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
@@ -1,7 +1,4 @@
|
|||||||
{
|
{
|
||||||
"files.exclude": {
|
|
||||||
"translations/**": true
|
|
||||||
},
|
|
||||||
"workbench.editor.enablePreview": false,
|
"workbench.editor.enablePreview": false,
|
||||||
"workbench.editor.enablePreviewFromQuickOpen": false
|
"workbench.editor.enablePreviewFromQuickOpen": false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
"ignore": [
|
"ignore": [
|
||||||
"assets",
|
"assets",
|
||||||
"script",
|
"script",
|
||||||
"translations",
|
|
||||||
"stylesheets",
|
"stylesheets",
|
||||||
"tests",
|
"tests",
|
||||||
"content",
|
"content",
|
||||||
|
|||||||
@@ -451,68 +451,6 @@ A helper that returns an array of files for a given path and file extension.
|
|||||||
---
|
---
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/fix-translation-errors.js`](i18n/fix-translation-errors.js)
|
|
||||||
|
|
||||||
Run this script to fix known frontmatter errors by copying values from english file Currently only fixing errors in: 'type', 'changelog' Please double check the changes created by this script before committing.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/homogenize-frontmatter.js`](i18n/homogenize-frontmatter.js)
|
|
||||||
|
|
||||||
Run this script to fix known frontmatter errors by copying values from english file Translatable properties are designated in the frontmatter JSON schema
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/lint-translation-files.js`](i18n/lint-translation-files.js)
|
|
||||||
|
|
||||||
Use this script as part of the translation merge process to output a list of either parsing or rendering errors in translated files and run script/i18n/reset-translated-file.js on them.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/msft-report-reset-files.js`](i18n/msft-report-reset-files.js)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/msft-reset-files-with-broken-liquid-tags.js`](i18n/msft-reset-files-with-broken-liquid-tags.js)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/msft-tokens.js`](i18n/msft-tokens.js)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/prune-stale-files.js`](i18n/prune-stale-files.js)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/reset-translated-file.js`](i18n/reset-translated-file.js)
|
|
||||||
|
|
||||||
This is a convenience script for replacing the contents of translated files with the English content from their corresponding source file.
|
|
||||||
|
|
||||||
Usage: script/i18n/reset-translated-file.js <filename>
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
$ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.md
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/test-html-pages.js`](i18n/test-html-pages.js)
|
### [`i18n/test-html-pages.js`](i18n/test-html-pages.js)
|
||||||
|
|
||||||
|
|
||||||
@@ -520,13 +458,6 @@ $ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.
|
|||||||
---
|
---
|
||||||
|
|
||||||
|
|
||||||
### [`i18n/test-render-translation.js`](i18n/test-render-translation.js)
|
|
||||||
|
|
||||||
Run this script to test-render all the translation files that have been changed (when compared to the `main` branch).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`kill-server-for-jest.js`](kill-server-for-jest.js)
|
### [`kill-server-for-jest.js`](kill-server-for-jest.js)
|
||||||
|
|
||||||
|
|
||||||
@@ -577,13 +508,6 @@ This script is intended to be used as a git "prepush" hook. If the current branc
|
|||||||
---
|
---
|
||||||
|
|
||||||
|
|
||||||
### [`prevent-translation-commits.js`](prevent-translation-commits.js)
|
|
||||||
|
|
||||||
This script is run as a git precommit hook (installed by husky after npm install). It detects changes to files the in the translations folder and prevents the commit if any changes exist.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
|
|
||||||
### [`purge-fastly`](purge-fastly)
|
### [`purge-fastly`](purge-fastly)
|
||||||
|
|
||||||
Run this script to manually purge the Fastly cache. Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file.
|
Run this script to manually purge the Fastly cache. Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file.
|
||||||
|
|||||||
@@ -2,6 +2,4 @@
|
|||||||
|
|
||||||
This directory stores scripts that modify content and/or data files. Because
|
This directory stores scripts that modify content and/or data files. Because
|
||||||
writers are updating content all the time, scripts in here require more
|
writers are updating content all the time, scripts in here require more
|
||||||
cross-team coordination and planning before they are run. Make sure to consider
|
cross-team coordination and planning before they are run. Make sure to consider if we can wait for the changes to come in through out translation automation.
|
||||||
whether a script added here also needs to be run on translation files or if we
|
|
||||||
can wait for the changes to come in through out translation automation.
|
|
||||||
|
|||||||
@@ -1,106 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
// [start-readme]
|
|
||||||
//
|
|
||||||
// Run this script to fix known frontmatter errors by copying values from english file
|
|
||||||
// Currently only fixing errors in: 'type', 'changelog'
|
|
||||||
// Please double check the changes created by this script before committing.
|
|
||||||
//
|
|
||||||
// [end-readme]
|
|
||||||
|
|
||||||
import path from 'path'
|
|
||||||
import { execSync } from 'child_process'
|
|
||||||
import { get, set } from 'lodash-es'
|
|
||||||
import fs from 'fs'
|
|
||||||
import fm from '../../lib/frontmatter.js'
|
|
||||||
import matter from 'gray-matter'
|
|
||||||
import chalk from 'chalk'
|
|
||||||
import yaml from 'js-yaml'
|
|
||||||
import releaseNotesSchema from '../../tests/helpers/schemas/release-notes-schema.js'
|
|
||||||
import revalidator from 'revalidator'
|
|
||||||
|
|
||||||
main()
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const fixableFmProps = Object.keys(fm.schema.properties)
|
|
||||||
.filter((property) => !fm.schema.properties[property].translatable)
|
|
||||||
.sort()
|
|
||||||
const fixableYmlProps = ['date']
|
|
||||||
|
|
||||||
const loadAndValidateContent = async (path, schema) => {
|
|
||||||
let fileContents
|
|
||||||
try {
|
|
||||||
fileContents = await fs.promises.readFile(path, 'utf8')
|
|
||||||
} catch (e) {
|
|
||||||
if (fs.existsSync(path)) {
|
|
||||||
console.error(e.message)
|
|
||||||
}
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
if (path.endsWith('yml')) {
|
|
||||||
let data
|
|
||||||
let errors = []
|
|
||||||
try {
|
|
||||||
data = yaml.load(fileContents)
|
|
||||||
} catch {}
|
|
||||||
if (data && schema) {
|
|
||||||
;({ errors } = revalidator.validate(data, schema))
|
|
||||||
}
|
|
||||||
return { data, errors, content: null }
|
|
||||||
} else {
|
|
||||||
return fm(fileContents)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const cmd =
|
|
||||||
'git -c diff.renameLimit=10000 diff --name-only origin/main | egrep "^translations/.*/(content/.+.md|data/release-notes/.*.yml)$"'
|
|
||||||
|
|
||||||
const maxBuffer = 1024 * 1024 * 2 // twice the default value
|
|
||||||
const changedFilesRelPaths = execSync(cmd, { maxBuffer }).toString().split('\n')
|
|
||||||
|
|
||||||
for (const relPath of changedFilesRelPaths) {
|
|
||||||
// Skip READMEs
|
|
||||||
if (!relPath || relPath.endsWith('README.md')) continue
|
|
||||||
|
|
||||||
// find the corresponding english file by removing the first 2 path segments: /translation/<language code>
|
|
||||||
const engAbsPath = relPath.split(path.sep).slice(2).join(path.sep)
|
|
||||||
|
|
||||||
const localisedResult = await loadAndValidateContent(relPath, releaseNotesSchema)
|
|
||||||
if (!localisedResult) continue
|
|
||||||
const { data, errors, content } = localisedResult
|
|
||||||
|
|
||||||
const fixableProps = relPath.endsWith('yml') ? fixableYmlProps : fixableFmProps
|
|
||||||
|
|
||||||
const fixableErrors = errors.filter(({ property }) => {
|
|
||||||
const prop = property.split('.')
|
|
||||||
return fixableProps.includes(prop[0])
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!data || fixableErrors.length === 0) continue
|
|
||||||
|
|
||||||
const engResult = await loadAndValidateContent(engAbsPath)
|
|
||||||
if (!engResult) continue
|
|
||||||
const { data: engData } = engResult
|
|
||||||
|
|
||||||
console.log(chalk.bold(relPath))
|
|
||||||
|
|
||||||
const newData = data
|
|
||||||
|
|
||||||
fixableErrors.forEach(({ property, message }) => {
|
|
||||||
const correctValue = get(engData, property)
|
|
||||||
console.log(chalk.red(` error message: [${property}] ${message}`))
|
|
||||||
console.log(` fix property [${property}]: ${get(data, property)} -> ${correctValue}`)
|
|
||||||
set(newData, property, correctValue)
|
|
||||||
})
|
|
||||||
|
|
||||||
let toWrite
|
|
||||||
if (content) {
|
|
||||||
toWrite = matter.stringify(content, newData, { lineWidth: 10000, forceQuotes: true })
|
|
||||||
} else {
|
|
||||||
toWrite = yaml.dump(newData, { lineWidth: 10000, forceQuotes: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
fs.writeFileSync(relPath, toWrite)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
// [start-readme]
|
|
||||||
//
|
|
||||||
// Run this script to fix known frontmatter errors by copying values from english file
|
|
||||||
// Translatable properties are designated in the frontmatter JSON schema
|
|
||||||
//
|
|
||||||
// [end-readme]
|
|
||||||
|
|
||||||
import path from 'path'
|
|
||||||
import fs from 'fs/promises'
|
|
||||||
import matter from 'gray-matter'
|
|
||||||
import walk from 'walk-sync'
|
|
||||||
import fm from '../../lib/frontmatter.js'
|
|
||||||
|
|
||||||
// Run!
|
|
||||||
main()
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const translatedMarkdownFiles = walk('translations')
|
|
||||||
.filter((filename) => {
|
|
||||||
return (
|
|
||||||
filename.includes('/content/') &&
|
|
||||||
filename.endsWith('.md') &&
|
|
||||||
!filename.endsWith('README.md')
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.map((filename) => `translations/${filename}`)
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
(
|
|
||||||
await Promise.all(
|
|
||||||
translatedMarkdownFiles.map(async (relPath) =>
|
|
||||||
updateTranslatedMarkdownFile(relPath).catch((e) => `Error in ${relPath}: ${e.message}`)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.filter(Boolean)
|
|
||||||
.join('\n')
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async function extractFrontmatter(path) {
|
|
||||||
const fileContents = await fs.readFile(path, 'utf8')
|
|
||||||
return fm(fileContents)
|
|
||||||
}
|
|
||||||
|
|
||||||
async function updateTranslatedMarkdownFile(relPath) {
|
|
||||||
// find the corresponding english file by removing the first 2 path segments: /translations/<language code>
|
|
||||||
const engAbsPath = relPath.split(path.sep).slice(2).join(path.sep)
|
|
||||||
|
|
||||||
// Load frontmatter from the source english file
|
|
||||||
let englishFrontmatter
|
|
||||||
try {
|
|
||||||
englishFrontmatter = await extractFrontmatter(engAbsPath)
|
|
||||||
} catch {
|
|
||||||
// This happens when an English file has been moved or deleted and translations are not in sync.
|
|
||||||
// It does mean this script will not homogenous those translated files, but the docs site does not
|
|
||||||
// load translated files that don't correlate to an English file, so those translated files can't break things.
|
|
||||||
// return `${relPath}: English file does not exist: ${engAbsPath}`
|
|
||||||
return // silence
|
|
||||||
}
|
|
||||||
|
|
||||||
const localisedFrontmatter = await extractFrontmatter(relPath)
|
|
||||||
if (!localisedFrontmatter) return `${relPath}: No localised frontmatter`
|
|
||||||
|
|
||||||
// Look for differences between the english and localised non-translatable properties
|
|
||||||
let overwroteSomething = false
|
|
||||||
for (const prop in localisedFrontmatter.data) {
|
|
||||||
if (
|
|
||||||
!fm.schema.properties[prop].translatable &&
|
|
||||||
englishFrontmatter.data[prop] &&
|
|
||||||
localisedFrontmatter.data[prop] !== englishFrontmatter.data[prop]
|
|
||||||
) {
|
|
||||||
localisedFrontmatter.data[prop] = englishFrontmatter.data[prop]
|
|
||||||
overwroteSomething = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// rewrite the localised file, if it changed
|
|
||||||
if (overwroteSomething) {
|
|
||||||
const toWrite = matter.stringify(localisedFrontmatter.content, localisedFrontmatter.data, {
|
|
||||||
lineWidth: 10000,
|
|
||||||
forceQuotes: true,
|
|
||||||
})
|
|
||||||
await fs.writeFile(relPath, toWrite)
|
|
||||||
|
|
||||||
// return `${relPath}: updated`
|
|
||||||
// silence
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
import { program } from 'commander'
|
|
||||||
import fs from 'fs'
|
|
||||||
import languages from '../../lib/languages.js'
|
|
||||||
|
|
||||||
const defaultWorkflowUrl = [
|
|
||||||
process.env.GITHUB_SERVER_URL,
|
|
||||||
process.env.GITHUB_REPOSITORY,
|
|
||||||
'actions/runs',
|
|
||||||
process.env.GITHUB_RUN_ID,
|
|
||||||
].join('/')
|
|
||||||
|
|
||||||
const reportTypes = {
|
|
||||||
'pull-request-body': pullRequestBodyReport,
|
|
||||||
csv: csvReport,
|
|
||||||
}
|
|
||||||
|
|
||||||
program
|
|
||||||
.description('Reads a translation batch log and generates a report')
|
|
||||||
.requiredOption('--language <language>', 'The language to compare')
|
|
||||||
.requiredOption('--log-file <log-file>', 'The batch log file')
|
|
||||||
.requiredOption(
|
|
||||||
'--report-type <report-type>',
|
|
||||||
'The batch log file, I.E: ' + Object.keys(reportTypes).join(', ')
|
|
||||||
)
|
|
||||||
.option('--workflow-url <workflow-url>', 'The workflow url', defaultWorkflowUrl)
|
|
||||||
.option('--csv-path <file-path>', 'The path to the CSV file')
|
|
||||||
.parse(process.argv)
|
|
||||||
|
|
||||||
const options = program.opts()
|
|
||||||
const language = languages[options.language]
|
|
||||||
const { logFile, workflowUrl, reportType, csvPath } = options
|
|
||||||
|
|
||||||
if (!Object.keys(reportTypes).includes(reportType)) {
|
|
||||||
throw new Error(`Invalid report type: ${reportType}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const logFileContents = fs.readFileSync(logFile, 'utf8')
|
|
||||||
|
|
||||||
const revertLines = logFileContents
|
|
||||||
.split('\n')
|
|
||||||
.filter((line) => line.match(/^(-> reverted to English)|^(-> removed)/))
|
|
||||||
.filter((line) => line.match(language.dir))
|
|
||||||
|
|
||||||
const reportEntries = revertLines.sort().map((line) => {
|
|
||||||
const [, file, reason] = line.match(/^-> (?:reverted to English|removed): (.*) Reason: (.*)$/)
|
|
||||||
return { file, reason }
|
|
||||||
})
|
|
||||||
|
|
||||||
function pullRequestBodyReport() {
|
|
||||||
return [
|
|
||||||
`New translation batch for ${language.name}. Product of [this workflow](${workflowUrl}).
|
|
||||||
|
|
||||||
## ${reportEntries.length} files reverted.
|
|
||||||
|
|
||||||
You can see the log in [\`${csvPath}\`](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/blob/${csvPath}).`,
|
|
||||||
].join('\n')
|
|
||||||
}
|
|
||||||
|
|
||||||
function csvReport() {
|
|
||||||
const lines = reportEntries.map(({ file, reason }) => {
|
|
||||||
return [file, reason].join(',')
|
|
||||||
})
|
|
||||||
|
|
||||||
return ['file,reason', lines].flat().join('\n')
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(reportTypes[reportType]())
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
import { program } from 'commander'
|
|
||||||
import { execFileSync } from 'child_process'
|
|
||||||
import { languageFiles, compareLiquidTags } from './msft-tokens.js'
|
|
||||||
import languages from '../../lib/languages.js'
|
|
||||||
|
|
||||||
program
|
|
||||||
.description('show-liquid-tags-diff')
|
|
||||||
.requiredOption('-l, --language <language>', 'The language to compare')
|
|
||||||
.option('-d, --dry-run', 'Just pretend to reset files')
|
|
||||||
.parse(process.argv)
|
|
||||||
|
|
||||||
function resetFiles(files) {
|
|
||||||
console.log(`Reseting ${files.length} files:`)
|
|
||||||
|
|
||||||
const dryRun = program.opts().dryRun ? '--dry-run' : ''
|
|
||||||
|
|
||||||
files.forEach((file) => {
|
|
||||||
const cmd = 'script/i18n/reset-translated-file.js'
|
|
||||||
const args = [file, '--reason', 'broken liquid tags', dryRun]
|
|
||||||
execFileSync(cmd, args, { stdio: 'inherit' })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
function deleteFiles(files) {
|
|
||||||
console.log(`Deleting ${files.length} files:`)
|
|
||||||
|
|
||||||
const dryRun = program.opts().dryRun ? '--dry-run' : ''
|
|
||||||
|
|
||||||
files.forEach((file) => {
|
|
||||||
const cmd = 'script/i18n/reset-translated-file.js'
|
|
||||||
const args = [
|
|
||||||
file,
|
|
||||||
'--remove',
|
|
||||||
'--reason',
|
|
||||||
'file deleted because it no longer exists in main',
|
|
||||||
dryRun,
|
|
||||||
]
|
|
||||||
execFileSync(cmd, args, { stdio: 'inherit' })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const options = program.opts()
|
|
||||||
const language = languages[options.language]
|
|
||||||
|
|
||||||
if (!language) {
|
|
||||||
throw new Error(`Language ${options.language} not found`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// languageFiles() returns an array indexed as follows:
|
|
||||||
// [0]: intersection of the files that exist in both main and the language-specific branch
|
|
||||||
// [1]: files that exist only in the language-specific branch, not in main
|
|
||||||
const allContentFiles = languageFiles(language, 'content')
|
|
||||||
const allDataFiles = languageFiles(language, 'data')
|
|
||||||
const files = [allContentFiles[0], allDataFiles[0]].flat()
|
|
||||||
const nonexitentFiles = [allContentFiles[1], allDataFiles[1]].flat()
|
|
||||||
const brokenFiles = []
|
|
||||||
|
|
||||||
files.forEach((file) => {
|
|
||||||
try {
|
|
||||||
// it throws error if the the syntax is invalid
|
|
||||||
const comparison = compareLiquidTags(file, language)
|
|
||||||
|
|
||||||
if (comparison.diff.count === 0) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
brokenFiles.push(comparison.translation)
|
|
||||||
} catch (e) {
|
|
||||||
brokenFiles.push(e.filePath)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
await resetFiles(brokenFiles)
|
|
||||||
await deleteFiles(nonexitentFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
main()
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
import walk from 'walk-sync'
|
|
||||||
import { Tokenizer } from 'liquidjs'
|
|
||||||
import { readFileSync } from 'fs'
|
|
||||||
import gitDiff from 'git-diff'
|
|
||||||
import _ from 'lodash'
|
|
||||||
|
|
||||||
function getGitDiff(a, b) {
|
|
||||||
return gitDiff(a, b, { flags: '--text --ignore-all-space' })
|
|
||||||
}
|
|
||||||
|
|
||||||
function getMissingLines(diff) {
|
|
||||||
return diff
|
|
||||||
.split('\n')
|
|
||||||
.filter((line) => line.startsWith('-'))
|
|
||||||
.map((line) => line.replace('-', ''))
|
|
||||||
}
|
|
||||||
|
|
||||||
function getExceedingLines(diff) {
|
|
||||||
return diff
|
|
||||||
.split('\n')
|
|
||||||
.filter((line) => line.startsWith('+'))
|
|
||||||
.map((line) => line.replace('+', ''))
|
|
||||||
}
|
|
||||||
|
|
||||||
export function languageFiles(language, folder = 'content') {
|
|
||||||
const englishFiles = walk(folder, { directories: false })
|
|
||||||
const languageFiles = walk(`${language.dir}/${folder}`, { directories: false })
|
|
||||||
return [
|
|
||||||
_.intersection(englishFiles, languageFiles).map((file) => `${folder}/${file}`),
|
|
||||||
_.difference(languageFiles, englishFiles).map((file) => `${language.dir}/${folder}/${file}`), // returns languageFiles not included in englishFiles
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
export function compareLiquidTags(file, language) {
|
|
||||||
const translation = `${language.dir}/${file}`
|
|
||||||
const sourceTokens = getTokensFromFile(file).rejectType('html')
|
|
||||||
const otherFileTokens = getTokensFromFile(translation).rejectType('html')
|
|
||||||
const diff = sourceTokens.diff(otherFileTokens)
|
|
||||||
|
|
||||||
return {
|
|
||||||
file,
|
|
||||||
translation,
|
|
||||||
diff,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getTokens(contents) {
|
|
||||||
const tokenizer = new Tokenizer(contents)
|
|
||||||
return new Tokens(...tokenizer.readTopLevelTokens())
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getTokensFromFile(filePath) {
|
|
||||||
const contents = readFileSync(filePath, 'utf8')
|
|
||||||
try {
|
|
||||||
return new Tokens(...getTokens(contents))
|
|
||||||
} catch (e) {
|
|
||||||
const error = new Error(`Error parsing ${filePath}: ${e.message}`)
|
|
||||||
error.filePath = filePath
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class Tokens extends Array {
|
|
||||||
rejectType(tagType) {
|
|
||||||
return this.filter(
|
|
||||||
(token) => token.constructor.name.toUpperCase() !== `${tagType}Token`.toUpperCase()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
onlyText() {
|
|
||||||
return this.map((token) => token.getText())
|
|
||||||
}
|
|
||||||
|
|
||||||
diff(otherTokens) {
|
|
||||||
const a = this.onlyText().sort()
|
|
||||||
const b = otherTokens.onlyText().sort()
|
|
||||||
|
|
||||||
const diff = getGitDiff(a.join('\n'), b.join('\n'))
|
|
||||||
|
|
||||||
if (!diff) {
|
|
||||||
return { count: 0, missing: [], exceeding: [], output: '' }
|
|
||||||
}
|
|
||||||
|
|
||||||
const missing = getMissingLines(diff)
|
|
||||||
const exceeding = getExceedingLines(diff)
|
|
||||||
const count = exceeding.length + missing.length
|
|
||||||
|
|
||||||
return { count, missing, exceeding, output: diff }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
import fs from 'fs'
|
|
||||||
import path from 'path'
|
|
||||||
import walk from 'walk-sync'
|
|
||||||
import { program } from 'commander'
|
|
||||||
import languages from '../../lib/languages.js'
|
|
||||||
|
|
||||||
program
|
|
||||||
.description(
|
|
||||||
`Removes any file in the translations directory that doesn't have a 1-1 mapping with an English file in the content directory`
|
|
||||||
)
|
|
||||||
.option('-d, --dry-run', `List the files that will be deleted, but don't remove them).`)
|
|
||||||
.parse(process.argv)
|
|
||||||
|
|
||||||
const languageDir = Object.keys(languages)
|
|
||||||
.filter((language) => !languages[language].wip && language !== 'en')
|
|
||||||
.map((language) => languages[language].dir)
|
|
||||||
|
|
||||||
main()
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const listOfContentFiles = walk(path.join(process.cwd(), 'content'), {
|
|
||||||
includeBasePath: false,
|
|
||||||
directories: false,
|
|
||||||
})
|
|
||||||
|
|
||||||
const translatedFilePaths = []
|
|
||||||
languageDir.forEach((directory) => {
|
|
||||||
const listOfFiles = walk(path.join(directory, 'content'), {
|
|
||||||
includeBasePath: true,
|
|
||||||
directories: false,
|
|
||||||
}).map((path) => path.replace(process.cwd(), ''))
|
|
||||||
translatedFilePaths.push(...listOfFiles)
|
|
||||||
})
|
|
||||||
|
|
||||||
let outOfSyncFilesCount = 0
|
|
||||||
translatedFilePaths.forEach((translatedFilePath) => {
|
|
||||||
const translationRelativePath = translatedFilePath.split('/content/')[1]
|
|
||||||
|
|
||||||
// If there is a 1:1 mapping of translated file to english file
|
|
||||||
// we're in sync, don't log
|
|
||||||
if (listOfContentFiles.includes(translationRelativePath)) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
outOfSyncFilesCount++
|
|
||||||
if (!program.opts().dryRun) {
|
|
||||||
fs.unlinkSync(translatedFilePath)
|
|
||||||
} else {
|
|
||||||
console.log(translatedFilePath)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
console.log(`Out of sync file size: ${outOfSyncFilesCount}`)
|
|
||||||
}
|
|
||||||
@@ -1,106 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
// [start-readme]
|
|
||||||
//
|
|
||||||
// This is a convenience script for replacing the contents of translated
|
|
||||||
// files with the English content from their corresponding source file.
|
|
||||||
//
|
|
||||||
// Usage:
|
|
||||||
// script/i18n/reset-translated-file.js <filename>
|
|
||||||
//
|
|
||||||
// Examples:
|
|
||||||
//
|
|
||||||
// $ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.md
|
|
||||||
//
|
|
||||||
// [end-readme]
|
|
||||||
|
|
||||||
import { program } from 'commander'
|
|
||||||
import { execSync } from 'child_process'
|
|
||||||
import assert from 'assert'
|
|
||||||
import fs from 'fs'
|
|
||||||
import path from 'path'
|
|
||||||
import chalk from 'chalk'
|
|
||||||
|
|
||||||
program
|
|
||||||
.description('reset translated files')
|
|
||||||
.option(
|
|
||||||
'-m, --prefer-main',
|
|
||||||
'Reset file to the translated file, try using the file from `main` branch first, if not found (usually due to renaming), fall back to English source.'
|
|
||||||
)
|
|
||||||
.option('-rm, --remove', 'Remove the translated files altogether')
|
|
||||||
.option('-d, --dry-run', 'Just pretend to reset files')
|
|
||||||
.option('-r, --reason <reason>', 'A reason why the file is getting reset')
|
|
||||||
.parse(process.argv)
|
|
||||||
|
|
||||||
const dryRun = program.opts().dryRun
|
|
||||||
const reason = program.opts().reason
|
|
||||||
const reasonMessage = reason ? `Reason: ${reason}` : ''
|
|
||||||
|
|
||||||
const resetToEnglishSource = (translationFilePath) => {
|
|
||||||
assert(
|
|
||||||
translationFilePath.startsWith('translations/'),
|
|
||||||
'path argument must be in the format `translations/<lang>/path/to/file`'
|
|
||||||
)
|
|
||||||
|
|
||||||
if (program.opts().remove) {
|
|
||||||
if (!dryRun) {
|
|
||||||
const fullPath = path.join(process.cwd(), translationFilePath)
|
|
||||||
fs.unlinkSync(fullPath)
|
|
||||||
}
|
|
||||||
console.log('-> removed: %s %s', translationFilePath, reasonMessage)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if (!fs.existsSync(translationFilePath)) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const relativePath = translationFilePath.split(path.sep).slice(2).join(path.sep)
|
|
||||||
const englishFile = path.join(process.cwd(), relativePath)
|
|
||||||
|
|
||||||
if (!dryRun && !fs.existsSync(englishFile)) {
|
|
||||||
fs.unlinkSync(translationFilePath)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!dryRun) {
|
|
||||||
// it is important to replace the file with English source instead of
|
|
||||||
// removing it, and relying on the fallback, because redired_from frontmatter
|
|
||||||
// won't work in fallbacks
|
|
||||||
const englishContent = fs.readFileSync(englishFile, 'utf8')
|
|
||||||
fs.writeFileSync(translationFilePath, englishContent)
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
'-> reverted to English: %s %s',
|
|
||||||
path.relative(process.cwd(), translationFilePath),
|
|
||||||
reasonMessage
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const [pathArg] = program.args
|
|
||||||
assert(pathArg, 'first arg must be a target filename')
|
|
||||||
|
|
||||||
// Is the arg a fully-qualified path?
|
|
||||||
const relativePath = fs.existsSync(pathArg) ? path.relative(process.cwd(), pathArg) : pathArg
|
|
||||||
|
|
||||||
if (program.opts().preferMain) {
|
|
||||||
try {
|
|
||||||
if (!dryRun) {
|
|
||||||
execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' })
|
|
||||||
}
|
|
||||||
console.log('-> reverted to file from main branch: %s %s', relativePath, reasonMessage)
|
|
||||||
} catch (e) {
|
|
||||||
if (e.message.includes('pathspec')) {
|
|
||||||
console.warn(
|
|
||||||
chalk.red(
|
|
||||||
`cannot find ${relativePath} in main branch (likely because it was renamed); falling back to English source file.`
|
|
||||||
)
|
|
||||||
)
|
|
||||||
resetToEnglishSource(relativePath)
|
|
||||||
} else {
|
|
||||||
console.warn(e.message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
resetToEnglishSource(relativePath)
|
|
||||||
}
|
|
||||||
@@ -1,124 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
// [start-readme]
|
|
||||||
//
|
|
||||||
// Run this script to test-render all the translation files that have been changed (when compared to the `main` branch).
|
|
||||||
//
|
|
||||||
// [end-readme]
|
|
||||||
|
|
||||||
import renderContent from '../../lib/render-content/index.js'
|
|
||||||
import loadSiteData from '../../lib/site-data.js'
|
|
||||||
import { loadPages } from '../../lib/page-data.js'
|
|
||||||
import languages from '../../lib/languages.js'
|
|
||||||
import { promisify } from 'util'
|
|
||||||
import ChildProcess, { execSync } from 'child_process'
|
|
||||||
import fs from 'fs'
|
|
||||||
import frontmatter from '../../lib/frontmatter.js'
|
|
||||||
import chalk from 'chalk'
|
|
||||||
import { YAMLException } from 'js-yaml'
|
|
||||||
|
|
||||||
const fmSchemaProperties = frontmatter.schema.properties
|
|
||||||
const exec = promisify(ChildProcess.exec)
|
|
||||||
|
|
||||||
main()
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const siteData = await loadAndPatchSiteData()
|
|
||||||
const pages = await loadPages()
|
|
||||||
const contextByLanguage = {}
|
|
||||||
for (const lang in languages) {
|
|
||||||
const langObj = languages[lang]
|
|
||||||
const [langCode] = langObj.dir === '' ? 'en' : langObj.dir.split('/').slice(1)
|
|
||||||
if (!langCode) continue
|
|
||||||
contextByLanguage[langCode] = {
|
|
||||||
site: siteData[langObj.code].site,
|
|
||||||
currentLanguage: langObj.code,
|
|
||||||
currentVersion: 'free-pro-team@latest',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const changedFilesRelPaths = execSync(
|
|
||||||
'git -c diff.renameLimit=10000 diff --name-only origin/main | egrep "^translations/.*/.+(.md|.yml)$"',
|
|
||||||
{ maxBuffer: 1024 * 1024 * 100 }
|
|
||||||
)
|
|
||||||
.toString()
|
|
||||||
.split('\n')
|
|
||||||
.filter((path) => path !== '' && !path.endsWith('README.md'))
|
|
||||||
.sort()
|
|
||||||
|
|
||||||
console.log(`Found ${changedFilesRelPaths.length} translated files.`)
|
|
||||||
|
|
||||||
for (const relPath of changedFilesRelPaths) {
|
|
||||||
const lang = relPath.split('/')[1]
|
|
||||||
const context = {
|
|
||||||
...contextByLanguage[lang],
|
|
||||||
pages,
|
|
||||||
page: pages.find((page) => {
|
|
||||||
const pageRelPath = `${languages[page.languageCode].dir}/content/${page.relativePath}`
|
|
||||||
return pageRelPath === relPath
|
|
||||||
}),
|
|
||||||
redirects: {},
|
|
||||||
}
|
|
||||||
|
|
||||||
// specifically test rendering data/variables files for broken liquid
|
|
||||||
if (relPath.includes('data/variables')) {
|
|
||||||
const fileContents = await fs.promises.readFile(relPath, 'utf8')
|
|
||||||
const { content } = frontmatter(fileContents)
|
|
||||||
|
|
||||||
try {
|
|
||||||
await renderContent.liquid.parseAndRender(content, context)
|
|
||||||
} catch (err) {
|
|
||||||
console.log(chalk.bold(relPath))
|
|
||||||
console.log(chalk.red(` error message: ${err.message}`))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!context.page && !relPath.includes('data/reusables')) continue
|
|
||||||
const fileContents = await fs.promises.readFile(relPath, 'utf8')
|
|
||||||
const { data, content } = frontmatter(fileContents)
|
|
||||||
const translatableFm = Object.keys(data).filter((key) => fmSchemaProperties[key].translatable)
|
|
||||||
try {
|
|
||||||
// test the content
|
|
||||||
await renderContent.liquid.parseAndRender(content, context)
|
|
||||||
// test each translatable frontmatter property
|
|
||||||
for (const key of translatableFm) {
|
|
||||||
await renderContent.liquid.parseAndRender(data[key], context)
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.log(chalk.bold(relPath))
|
|
||||||
console.log(chalk.red(` error message: ${err.message}`))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadAndPatchSiteData(filesWithKnownIssues = {}) {
|
|
||||||
try {
|
|
||||||
const siteData = loadSiteData()
|
|
||||||
return siteData
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof YAMLException && error.mark) {
|
|
||||||
const relPath = error.mark.name
|
|
||||||
if (!filesWithKnownIssues[relPath]) {
|
|
||||||
// Note the file as problematic
|
|
||||||
filesWithKnownIssues[relPath] = true
|
|
||||||
|
|
||||||
// This log is important as it will get ${relPath} written to a logfile
|
|
||||||
console.log(chalk.bold(relPath))
|
|
||||||
console.log(chalk.red(` error message: ${error.toString()}`))
|
|
||||||
|
|
||||||
// Reset the file
|
|
||||||
console.warn(`resetting file "${relPath}" due to loadSiteData error: ${error.toString()}`)
|
|
||||||
await exec(
|
|
||||||
`script/i18n/reset-translated-file.js --prefer-main ${relPath} --reason="loadSiteData error"`
|
|
||||||
)
|
|
||||||
|
|
||||||
// Try to load the site data again
|
|
||||||
return loadAndPatchSiteData(filesWithKnownIssues)
|
|
||||||
} else {
|
|
||||||
console.error(`FATAL: Tried to reset file "${relPath}" but still had errors`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,4 +8,6 @@
|
|||||||
|
|
||||||
source script/check-for-node
|
source script/check-for-node
|
||||||
|
|
||||||
|
# TODO would need git clones from the language repos
|
||||||
|
|
||||||
npm run start-all-languages
|
npm run start-all-languages
|
||||||
|
|||||||
@@ -1,82 +0,0 @@
|
|||||||
import { expect } from '@jest/globals'
|
|
||||||
import path from 'path'
|
|
||||||
import { fileURLToPath } from 'url'
|
|
||||||
|
|
||||||
import { getTokensFromFile, Tokens } from '../../../script/i18n/msft-tokens'
|
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
|
||||||
|
|
||||||
function getFixturePath(name) {
|
|
||||||
return path.join(__dirname, '../..', 'fixtures', name)
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('getTokensFromFile', () => {
|
|
||||||
let fixturePath
|
|
||||||
let tokens
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
fixturePath = getFixturePath('liquid-tags/minimal-conditional.md')
|
|
||||||
tokens = getTokensFromFile(fixturePath)
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('getTokensFromFile', () => {
|
|
||||||
it('returns all the tokens from a template file', () => {
|
|
||||||
expect(tokens.length).toEqual(7)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('Tokens', () => {
|
|
||||||
describe('.rejectType', () => {
|
|
||||||
it('rejects tokens of a particular type', () => {
|
|
||||||
const nonHtmlTokens = tokens.rejectType('html')
|
|
||||||
|
|
||||||
expect(nonHtmlTokens.length).toEqual(3)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('.diff', () => {
|
|
||||||
let tokens
|
|
||||||
let otherTokens
|
|
||||||
let reverseTokens
|
|
||||||
|
|
||||||
const addTokens = (collection, elements) => {
|
|
||||||
elements.forEach((element) => {
|
|
||||||
collection.push({ getText: () => element })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
tokens = new Tokens()
|
|
||||||
otherTokens = new Tokens()
|
|
||||||
reverseTokens = new Tokens()
|
|
||||||
addTokens(tokens, ['apples', 'bananas', 'oranges'])
|
|
||||||
addTokens(otherTokens, ['apples', 'oranges'])
|
|
||||||
addTokens(reverseTokens, ['oranges', 'bananas', 'apples'])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('shows elements that are missing', () => {
|
|
||||||
const diff = tokens.diff(otherTokens)
|
|
||||||
|
|
||||||
expect(diff.count).toEqual(1)
|
|
||||||
expect(diff.missing).toEqual(['bananas'])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('shows elements that are exceeding', () => {
|
|
||||||
const diff = otherTokens.diff(tokens)
|
|
||||||
|
|
||||||
expect(diff.count).toEqual(1)
|
|
||||||
expect(diff.exceeding).toEqual(['bananas'])
|
|
||||||
})
|
|
||||||
|
|
||||||
it('shows no difference when collections are the same', () => {
|
|
||||||
const diff = tokens.diff(tokens)
|
|
||||||
|
|
||||||
expect(diff.count).toEqual(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('shows no difference when tokens are in different order', () => {
|
|
||||||
const diff = tokens.diff(reverseTokens)
|
|
||||||
expect(diff.count).toEqual(0)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
Reference in New Issue
Block a user