diff --git a/.github/workflows/create-translation-batch-pr.yml b/.github/workflows/create-translation-batch-pr.yml index 6154870d67..b60933fc98 100644 --- a/.github/workflows/create-translation-batch-pr.yml +++ b/.github/workflows/create-translation-batch-pr.yml @@ -89,7 +89,7 @@ jobs: - name: Commit crowdin sync run: | - git add . + git add translations/${{ matrix.language }} git commit -m "Add crowdin translations" || echo "Nothing to commit" - name: 'Setup node' @@ -102,13 +102,13 @@ jobs: - name: Reset files with broken liquid tags run: | node script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language_code }} - git add . && git commit -m "run script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language_code }}" || echo "Nothing to commit" + git add translations/${{ matrix.language }} && git commit -m "run script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language_code }}" || echo "Nothing to commit" # step 5 in docs-engineering/crowdin.md using script from docs-internal#22709 - name: Reset known broken files run: | node script/i18n/reset-known-broken-translation-files.js - git add . && git commit -m "run script/i18n/reset-known-broken-translation-files.js" || echo "Nothing to commit" + git add translations/${{ matrix.language }} && git commit -m "run script/i18n/reset-known-broken-translation-files.js" || echo "Nothing to commit" env: GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }} @@ -116,25 +116,32 @@ jobs: - name: Homogenize frontmatter run: | node script/i18n/homogenize-frontmatter.js - git add . && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit" + git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit" # step 7 in docs-engineering/crowdin.md - name: Fix translation errors run: | node script/i18n/fix-translation-errors.js - git add . && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit" + git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit" # step 8a in docs-engineering/crowdin.md - name: Check parsing run: | node script/i18n/lint-translation-files.js --check parsing - git add . && git commit -m "Run script/i18n/lint-translation-files.js --check parsing" || echo "Nothing to commit" + git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/lint-translation-files.js --check parsing" || echo "Nothing to commit" # step 8b in docs-engineering/crowdin.md - name: Check rendering run: | node script/i18n/lint-translation-files.js --check rendering - git add . && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit" + git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit" + + - name: Check in CSV report + run: | + mkdir -p log + csvFile=log/${{ matrix.language_code }}-resets.csv + script/i18n/report-reset-files.js --report-type=csv --language=${{ matrix.language_code }} --log-file=/tmp/batch.log > $csvFile + git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit" - name: Create Pull Request env: @@ -142,11 +149,12 @@ jobs: # We'll try to create the pull request based on the changes we pushed up in the branch. # If there are actually no differences between the branch and `main`, we'll delete it. run: | + script/i18n/report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language_code }} --log-file=/tmp/batch.log > /tmp/pr-body.txt git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }} - gh pr create -t "New translation batch for ${{ matrix.language }}" \ + gh pr create --title "New translation batch for ${{ matrix.language }}" \ --base=main \ --head=${{ steps.set-branch.outputs.BRANCH_NAME }} \ - -b "New batch for ${{ matrix.language }} created by [this workflow]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" || git push origin :${{ steps.set-branch.outputs.BRANCH_NAME }} + --body-file /tmp/pr-body.txt || git push origin :${{ steps.set-branch.outputs.BRANCH_NAME }} # When the maximum execution time is reached for this job, Actions cancels the workflow run. # This emits a notification for the first responder to triage. diff --git a/script/i18n/fix-translation-errors.js b/script/i18n/fix-translation-errors.js index 8c18781474..a76813989d 100755 --- a/script/i18n/fix-translation-errors.js +++ b/script/i18n/fix-translation-errors.js @@ -33,7 +33,9 @@ async function main() { try { fileContents = await readFileAsync(path, 'utf8') } catch (e) { - console.error(e.message) + if (fs.existsSync(path)) { + console.error(e.message) + } return null } diff --git a/script/i18n/lint-translation-files.js b/script/i18n/lint-translation-files.js index 2c3cc63aa8..df62d009fa 100755 --- a/script/i18n/lint-translation-files.js +++ b/script/i18n/lint-translation-files.js @@ -79,7 +79,9 @@ function lintAndResetFiles(checkType) { // We are not passing --prefer-main because we want to remove the file so we // reset it directly to the English source filesToReset.forEach((file) => { - execSync(`script/i18n/reset-translated-file.js ${file}`) + execSync(`script/i18n/reset-translated-file.js ${file} --reason="${checkType} error"`, { + stdio: 'inherit', + }) }) // Print a message with next steps diff --git a/script/i18n/report-reset-files.js b/script/i18n/report-reset-files.js new file mode 100755 index 0000000000..6ed9a165af --- /dev/null +++ b/script/i18n/report-reset-files.js @@ -0,0 +1,85 @@ +#!/usr/bin/env node + +import program from 'commander' +import fs from 'fs' +import languages from '../../lib/languages.js' + +const defaultWorkflowUrl = [ + process.env.GITHUB_SERVER_URL, + process.env.GITHUB_REPOSITORY, + 'actions/runs', + process.env.GITHUB_RUN_ID, +].join('/') + +const reportTypes = { + 'pull-request-body': pullRequestBodyReport, + csv: csvReport, +} + +program + .description('Reads a translation batch log and generates a report') + .requiredOption('--language ', 'The language to compare') + .requiredOption('--log-file ', 'The batch log file') + .requiredOption( + '--report-type ', + 'The batch log file, I.E: ' + Object.keys(reportTypes).join(', ') + ) + .option('--workflow-url ', 'The workflow url', defaultWorkflowUrl) + .parse(process.argv) + +const options = program.opts() +const language = languages[options.language] +const { logFile, workflowUrl, reportType } = options + +if (!Object.keys(reportTypes).includes(reportType)) { + throw new Error(`Invalid report type: ${reportType}`) +} + +const logFileContents = fs.readFileSync(logFile, 'utf8') + +const revertLines = logFileContents + .split('\n') + .filter((line) => line.match(/^-> reverted to English/)) + .filter((line) => line.match(language.dir)) + +const reportEntries = revertLines.sort().map((line) => { + const [, file, reason] = line.match(/^-> reverted to English: (.*) Reason: (.*)$/) + return { file, reason } +}) + +function pullRequestBodyReport() { + const body = [ + `New translation batch for ${language.name}. Product of [this workflow](${workflowUrl}).`, + '\n', + `## ${reportEntries.length} files reverted.`, + ] + + const filesByReason = {} + + reportEntries.forEach(({ file, reason }) => { + filesByReason[reason] = filesByReason[reason] || [] + filesByReason[reason].push(file) + }) + + Object.keys(filesByReason) + .sort() + .forEach((reason) => { + const files = filesByReason[reason] + body.push(`\n### ${reason}`) + body.push(`\n${files.length} files:\n`) + const checkBoxes = files.map((file) => `- [ ] ${file}`) + body.push(checkBoxes) + }) + + return body.join('\n') +} + +function csvReport() { + const lines = reportEntries.map(({ file, reason }) => { + return [file, reason].join(',') + }) + + return ['file,reason', lines].flat().join('\n') +} + +console.log(reportTypes[reportType]()) diff --git a/script/i18n/reset-known-broken-translation-files.js b/script/i18n/reset-known-broken-translation-files.js index 4fc99e4842..c1cd5a34d9 100755 --- a/script/i18n/reset-known-broken-translation-files.js +++ b/script/i18n/reset-known-broken-translation-files.js @@ -50,7 +50,10 @@ async function main() { // This is done sequentially to ensure only one Git operation is running at any given time. brokenFilesArray.forEach((file) => { console.log(`Resetting ${file}`) - execSync(`node script/i18n/reset-translated-file.js ${file}`) + execSync( + `script/i18n/reset-translated-file.js ${file} --reason="Listed in localization-support#489"`, + { stdio: 'inherit' } + ) }) // Print a message with next steps. diff --git a/script/i18n/reset-translated-file.js b/script/i18n/reset-translated-file.js index 55479da0d0..952e7fbd46 100755 --- a/script/i18n/reset-translated-file.js +++ b/script/i18n/reset-translated-file.js @@ -30,8 +30,14 @@ program '-m, --prefer-main', 'Reset file to the translated file, try using the file from `main` branch first, if not found (usually due to renaming), fall back to English source.' ) + .option('-d, --dry-run', 'Just pretend to reset files') + .option('-r, --reason ', 'A reason why the file is getting reset') .parse(process.argv) +const dryRun = program.opts().dryRun +const reason = program.opts().reason +const reasonMessage = reason ? `Reason: ${reason}` : '' + const resetToEnglishSource = (translationFilePath) => { assert( translationFilePath.startsWith('translations/'), @@ -45,15 +51,21 @@ const resetToEnglishSource = (translationFilePath) => { const relativePath = translationFilePath.split(path.sep).slice(2).join(path.sep) const englishFile = path.join(process.cwd(), relativePath) - if (!fs.existsSync(englishFile)) { + if (!dryRun && !fs.existsSync(englishFile)) { fs.unlinkSync(translationFilePath) return } - // replace file with English source - const englishContent = fs.readFileSync(englishFile, 'utf8') - fs.writeFileSync(translationFilePath, englishContent) - console.log('-> reverted to English: %s', path.relative(process.cwd(), translationFilePath)) + if (!dryRun) { + // replace file with English source + const englishContent = fs.readFileSync(englishFile, 'utf8') + fs.writeFileSync(translationFilePath, englishContent) + } + console.log( + '-> reverted to English: %s %s', + path.relative(process.cwd(), translationFilePath), + reasonMessage + ) } const [pathArg] = program.args @@ -64,8 +76,10 @@ const relativePath = fs.existsSync(pathArg) ? path.relative(process.cwd(), pathA if (program.opts().preferMain) { try { - execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' }) - console.log('-> reverted to file from main branch: %s', relativePath) + if (!dryRun) { + execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' }) + } + console.log('-> reverted to file from main branch: %s %s', relativePath, reasonMessage) } catch (e) { if (e.message.includes('pathspec')) { console.warn( diff --git a/script/i18n/test-render-translation.js b/script/i18n/test-render-translation.js index 39268738fc..2b9fc06c89 100755 --- a/script/i18n/test-render-translation.js +++ b/script/i18n/test-render-translation.js @@ -94,7 +94,9 @@ async function loadAndPatchSiteData(filesWithKnownIssues = {}) { // Reset the file console.warn(`resetting file "${relPath}" due to loadSiteData error: ${error.toString()}`) - await exec(`script/i18n/reset-translated-file.js --prefer-main ${relPath}`) + await exec( + `script/i18n/reset-translated-file.js --prefer-main ${relPath} --reason="loadSiteData error"` + ) // Try to load the site data again return loadAndPatchSiteData(filesWithKnownIssues)