Merge pull request #12483 from github/repo-sync

repo sync

.github/workflows/create-translation-batch-pr.yml

@@ -89,7 +89,7 @@ jobs:
 
       - name: Commit crowdin sync
         run: |
-          git add .
+          git add translations/${{ matrix.language }}
           git commit -m "Add crowdin translations" || echo "Nothing to commit"
 
       - name: 'Setup node'
@@ -102,13 +102,13 @@ jobs:
       - name: Reset files with broken liquid tags
         run: |
           node script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language_code }}
-          git add . && git commit -m "run script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language_code }}" || echo "Nothing to commit"
+          git add translations/${{ matrix.language }} && git commit -m "run script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language_code }}" || echo "Nothing to commit"
 
       # step 5 in docs-engineering/crowdin.md using script from docs-internal#22709
       - name: Reset known broken files
         run: |
           node script/i18n/reset-known-broken-translation-files.js
-          git add . && git commit -m "run script/i18n/reset-known-broken-translation-files.js" || echo "Nothing to commit"
+          git add translations/${{ matrix.language }} && git commit -m "run script/i18n/reset-known-broken-translation-files.js" || echo "Nothing to commit"
         env:
           GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }}
 
@@ -116,25 +116,32 @@ jobs:
       - name: Homogenize frontmatter
         run: |
           node script/i18n/homogenize-frontmatter.js
-          git add . && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit"
+          git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit"
 
       # step 7 in docs-engineering/crowdin.md
       - name: Fix translation errors
         run: |
           node script/i18n/fix-translation-errors.js
-          git add . && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit"
+          git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit"
 
       # step 8a in docs-engineering/crowdin.md
       - name: Check parsing
         run: |
           node script/i18n/lint-translation-files.js --check parsing
-          git add . && git commit -m "Run script/i18n/lint-translation-files.js --check parsing" || echo "Nothing to commit"
+          git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/lint-translation-files.js --check parsing" || echo "Nothing to commit"
 
       # step 8b in docs-engineering/crowdin.md
       - name: Check rendering
         run: |
           node script/i18n/lint-translation-files.js --check rendering
-          git add . && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit"
+          git add translations/${{ matrix.language }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit"
+
+      - name: Check in CSV report
+        run: |
+          mkdir -p log
+          csvFile=log/${{ matrix.language_code }}-resets.csv
+          script/i18n/report-reset-files.js --report-type=csv --language=${{ matrix.language_code }} --log-file=/tmp/batch.log > $csvFile
+          git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit"
 
       - name: Create Pull Request
         env:
@@ -142,11 +149,12 @@ jobs:
         # We'll try to create the pull request based on the changes we pushed up in the branch.
         # If there are actually no differences between the branch and `main`, we'll delete it.
         run: |
+          script/i18n/report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language_code }} --log-file=/tmp/batch.log > /tmp/pr-body.txt
           git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }}
-          gh pr create -t "New translation batch for ${{ matrix.language }}" \
+          gh pr create --title "New translation batch for ${{ matrix.language }}" \
             --base=main \
             --head=${{ steps.set-branch.outputs.BRANCH_NAME }} \
-            -b "New batch for ${{ matrix.language }} created by [this workflow]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" ||  git push origin :${{ steps.set-branch.outputs.BRANCH_NAME }}
+            --body-file /tmp/pr-body.txt || git push origin :${{ steps.set-branch.outputs.BRANCH_NAME }}
 
       # When the maximum execution time is reached for this job, Actions cancels the workflow run.
       # This emits a notification for the first responder to triage.

script/i18n/fix-translation-errors.js

@@ -33,7 +33,9 @@ async function main() {
     try {
       fileContents = await readFileAsync(path, 'utf8')
     } catch (e) {
-      console.error(e.message)
+      if (fs.existsSync(path)) {
+        console.error(e.message)
+      }
       return null
     }
 

script/i18n/lint-translation-files.js

@@ -79,7 +79,9 @@ function lintAndResetFiles(checkType) {
   // We are not passing --prefer-main because we want to remove the file so we
   // reset it directly to the English source
   filesToReset.forEach((file) => {
-    execSync(`script/i18n/reset-translated-file.js ${file}`)
+    execSync(`script/i18n/reset-translated-file.js ${file} --reason="${checkType} error"`, {
+      stdio: 'inherit',
+    })
   })
 
   // Print a message with next steps

script/i18n/report-reset-files.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+
+import program from 'commander'
+import fs from 'fs'
+import languages from '../../lib/languages.js'
+
+const defaultWorkflowUrl = [
+  process.env.GITHUB_SERVER_URL,
+  process.env.GITHUB_REPOSITORY,
+  'actions/runs',
+  process.env.GITHUB_RUN_ID,
+].join('/')
+
+const reportTypes = {
+  'pull-request-body': pullRequestBodyReport,
+  csv: csvReport,
+}
+
+program
+  .description('Reads a translation batch log and generates a report')
+  .requiredOption('--language <language>', 'The language to compare')
+  .requiredOption('--log-file <log-file>', 'The batch log file')
+  .requiredOption(
+    '--report-type <report-type>',
+    'The batch log file, I.E: ' + Object.keys(reportTypes).join(', ')
+  )
+  .option('--workflow-url <workflow-url>', 'The workflow url', defaultWorkflowUrl)
+  .parse(process.argv)
+
+const options = program.opts()
+const language = languages[options.language]
+const { logFile, workflowUrl, reportType } = options
+
+if (!Object.keys(reportTypes).includes(reportType)) {
+  throw new Error(`Invalid report type: ${reportType}`)
+}
+
+const logFileContents = fs.readFileSync(logFile, 'utf8')
+
+const revertLines = logFileContents
+  .split('\n')
+  .filter((line) => line.match(/^-> reverted to English/))
+  .filter((line) => line.match(language.dir))
+
+const reportEntries = revertLines.sort().map((line) => {
+  const [, file, reason] = line.match(/^-> reverted to English: (.*) Reason: (.*)$/)
+  return { file, reason }
+})
+
+function pullRequestBodyReport() {
+  const body = [
+    `New translation batch for ${language.name}. Product of [this workflow](${workflowUrl}).`,
+    '\n',
+    `## ${reportEntries.length} files reverted.`,
+  ]
+
+  const filesByReason = {}
+
+  reportEntries.forEach(({ file, reason }) => {
+    filesByReason[reason] = filesByReason[reason] || []
+    filesByReason[reason].push(file)
+  })
+
+  Object.keys(filesByReason)
+    .sort()
+    .forEach((reason) => {
+      const files = filesByReason[reason]
+      body.push(`\n### ${reason}`)
+      body.push(`\n${files.length} files:\n`)
+      const checkBoxes = files.map((file) => `- [ ] ${file}`)
+      body.push(checkBoxes)
+    })
+
+  return body.join('\n')
+}
+
+function csvReport() {
+  const lines = reportEntries.map(({ file, reason }) => {
+    return [file, reason].join(',')
+  })
+
+  return ['file,reason', lines].flat().join('\n')
+}
+
+console.log(reportTypes[reportType]())

script/i18n/reset-known-broken-translation-files.js

@@ -50,7 +50,10 @@ async function main() {
   // This is done sequentially to ensure only one Git operation is running at any given time.
   brokenFilesArray.forEach((file) => {
     console.log(`Resetting ${file}`)
-    execSync(`node script/i18n/reset-translated-file.js ${file}`)
+    execSync(
+      `script/i18n/reset-translated-file.js ${file} --reason="Listed in localization-support#489"`,
+      { stdio: 'inherit' }
+    )
   })
 
   // Print a message with next steps.

script/i18n/reset-translated-file.js

@@ -30,8 +30,14 @@ program
     '-m, --prefer-main',
     'Reset file to the translated file, try using the file from `main` branch first, if not found (usually due to renaming), fall back to English source.'
   )
+  .option('-d, --dry-run', 'Just pretend to reset files')
+  .option('-r, --reason <reason>', 'A reason why the file is getting reset')
   .parse(process.argv)
 
+const dryRun = program.opts().dryRun
+const reason = program.opts().reason
+const reasonMessage = reason ? `Reason: ${reason}` : ''
+
 const resetToEnglishSource = (translationFilePath) => {
   assert(
     translationFilePath.startsWith('translations/'),
@@ -45,15 +51,21 @@ const resetToEnglishSource = (translationFilePath) => {
   const relativePath = translationFilePath.split(path.sep).slice(2).join(path.sep)
   const englishFile = path.join(process.cwd(), relativePath)
 
-  if (!fs.existsSync(englishFile)) {
+  if (!dryRun && !fs.existsSync(englishFile)) {
     fs.unlinkSync(translationFilePath)
     return
   }
 
-  // replace file with English source
-  const englishContent = fs.readFileSync(englishFile, 'utf8')
-  fs.writeFileSync(translationFilePath, englishContent)
-  console.log('-> reverted to English: %s', path.relative(process.cwd(), translationFilePath))
+  if (!dryRun) {
+    // replace file with English source
+    const englishContent = fs.readFileSync(englishFile, 'utf8')
+    fs.writeFileSync(translationFilePath, englishContent)
+  }
+  console.log(
+    '-> reverted to English: %s %s',
+    path.relative(process.cwd(), translationFilePath),
+    reasonMessage
+  )
 }
 
 const [pathArg] = program.args
@@ -64,8 +76,10 @@ const relativePath = fs.existsSync(pathArg) ? path.relative(process.cwd(), pathA
 
 if (program.opts().preferMain) {
   try {
-    execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' })
-    console.log('-> reverted to file from main branch: %s', relativePath)
+    if (!dryRun) {
+      execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' })
+    }
+    console.log('-> reverted to file from main branch: %s %s', relativePath, reasonMessage)
   } catch (e) {
     if (e.message.includes('pathspec')) {
       console.warn(

script/i18n/test-render-translation.js

@@ -94,7 +94,9 @@ async function loadAndPatchSiteData(filesWithKnownIssues = {}) {
 
         // Reset the file
         console.warn(`resetting file "${relPath}" due to loadSiteData error: ${error.toString()}`)
-        await exec(`script/i18n/reset-translated-file.js --prefer-main ${relPath}`)
+        await exec(
+          `script/i18n/reset-translated-file.js --prefer-main ${relPath} --reason="loadSiteData error"`
+        )
 
         // Try to load the site data again
         return loadAndPatchSiteData(filesWithKnownIssues)