From 8b135ca1b55cc1d044f89d4fd728f8638db66e34 Mon Sep 17 00:00:00 2001 From: Dimitry Kolyshev Date: Mon, 29 Jun 2026 03:15:13 +0000 Subject: [PATCH] Pull request: AGDNS-4161-imp-translation-scripts Squashed commit of the following: commit 8a79a364cf0614a3c7ccef6efcfaae546a6087c2 Merge: 70bb94848 5c4af121d Author: Dimitry Kolyshev Date: Mon Jun 29 09:17:02 2026 +0700 Merge remote-tracking branch 'origin/master' into AGDNS-4161-imp-translation-scripts commit 70bb9484864405bc1776af1138578d90bdea76e1 Merge: aa62ca71f 07d8d77a5 Author: Dimitry Kolyshev Date: Fri Jun 26 08:50:13 2026 +0700 Merge remote-tracking branch 'origin/master' into AGDNS-4161-imp-translation-scripts commit aa62ca71f859f359eb5db1d2c354afc07b4e1369 Author: Dimitry Kolyshev Date: Fri Jun 26 08:49:55 2026 +0700 scripts: imp docs commit 4195a5dbbc73a2bef5548aa0ce54709f566e1608 Author: Dimitry Kolyshev Date: Thu Jun 25 10:59:20 2026 +0700 scripts: fix translations commit be40c845cd86e21f3a4a08982ce6d8801149e643 Author: Dimitry Kolyshev Date: Thu Jun 25 10:55:55 2026 +0700 scripts: imp code commit 9fa78efe7115be8496cf89d40dbcc8e7a4220cbc Author: Dimitry Kolyshev Date: Thu Jun 25 10:16:08 2026 +0700 scripts: imp code commit ad2cbc31285dba5a870098945cded8a22ee1418c Author: Dimitry Kolyshev Date: Wed Jun 24 10:46:53 2026 +0700 scripts: imp code --- .twosky.json | 8 ++ scripts/README.md | 6 +- scripts/translations/download.go | 6 +- scripts/translations/main.go | 124 +++++++++++++++---------------- scripts/translations/twosky.go | 84 ++++++++++++--------- scripts/translations/upload.go | 2 +- 6 files changed, 122 insertions(+), 108 deletions(-) diff --git a/.twosky.json b/.twosky.json index 31c27f104a2..7a40c489259 100644 --- a/.twosky.json +++ b/.twosky.json @@ -42,6 +42,10 @@ "zh-cn": "简体中文", "zh-hk": "繁體中文（香港）", "zh-tw": "正體中文（台灣）" + }, + "metadata": { + "locales_dir": "./client/src/__locales", + "sources_dir": "./client/src" } }, { @@ -87,6 +91,10 @@ "zh-cn": "简体中文", "zh-hk": "繁體中文（香港）", "zh-tw": "正體中文（台灣）" + }, + "metadata": { + "locales_dir": "./client/src/__locales-services", + "sources_dir": "./client/src" } } ] diff --git a/scripts/README.md b/scripts/README.md index bdd839b8324..a06edb2119f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -206,7 +206,7 @@ Optional environment: - `go run ./scripts/translations auto-add`: add locales with additions to the git and restore locales with deletions. -After the download you’ll find the output locales in the `client/src/__locales/` directory. +After the download you’ll find the output locales in the configured directory. Optional environment: @@ -216,9 +216,7 @@ Optional environment: - `TWOSKY_URI`: set an alternative URL for `download` or `upload`. -- `TWOSKY_PROJECT_ID`: set an alternative project ID for `download` or `upload`. - - Deprectated: This environment variable should not be used since the script began supporting multiple configurations. +- `TWOSKY_PROJECT_ID`: set an alternative project ID. The default value is `home`. ## `companiesdb/`: Whotracks.me database converter diff --git a/scripts/translations/download.go b/scripts/translations/download.go index 383250dffcb..8be27e14569 100644 --- a/scripts/translations/download.go +++ b/scripts/translations/download.go @@ -49,15 +49,15 @@ func (c *twoskyClient) download(ctx context.Context, l *slog.Logger) { wg.Go(dw.run) } - for _, baseFile := range c.localizableFiles { - dir, file := filepath.Split(baseFile) + for _, mask := range c.localizableFiles { + _, file := filepath.Split(mask) for _, lang := range c.langs { uri := translationURL(downloadURI, file, c.projectID, lang) reqCh <- downloadRequest{ uri: uri, - dir: dir, + dir: c.localesDir, } } } diff --git a/scripts/translations/main.go b/scripts/translations/main.go index fff8bf3f224..71a272b9a0d 100644 --- a/scripts/translations/main.go +++ b/scripts/translations/main.go @@ -26,11 +26,10 @@ import ( // TODO(e.burkov): Remove the default as they should be set by configuration. const ( - twoskyConfFile = "./.twosky.json" - localesDirHome = "./client/src/__locales" - defaultBaseFile = "en.json" - srcDir = "./client/src" - twoskyURI = "https://twosky.int.agrd.dev/api/v1" + defaultBaseFile = "en.json" + defaultProjectID = "home" + twoskyConfFile = "./.twosky.json" + twoskyURI = "https://twosky.int.agrd.dev/api/v1" readLimit = 1 * datasize.MB uploadTimeout = 20 * time.Second @@ -76,27 +75,20 @@ func main() { usage("") } - homeConf, servicesConf, err := readTwoskyConfig() - errors.Check(err) - - var cli *twoskyClient + conf := errors.Must(newTwoskyConfig()) + cli := errors.Must(newTwoskyClient(conf)) switch os.Args[1] { case "summary": - errors.Check(summary(homeConf.Languages)) + errors.Check(cli.summary()) case "download": - cli = errors.Must(newTwoskyClient(homeConf)) - cli.download(ctx, l) - - cli = errors.Must(newTwoskyClient(servicesConf)) cli.download(ctx, l) case "unused": - errors.Check(unused(ctx, l, homeConf.LocalizableFiles[0])) + errors.Check(cli.unused(ctx, l)) case "upload": - cli = errors.Must(newTwoskyClient(homeConf)) errors.Check(cli.upload()) case "auto-add": - errors.Check(autoAdd(ctx, l, homeConf.LocalizableFiles[0])) + errors.Check(cli.autoAdd(ctx, l)) default: usage("unknown command") } @@ -151,10 +143,10 @@ func validateLanguageStr(str string, all languages) (langs []langCode, err error return langs, nil } -// readLocales reads file with name fn and returns a map, where key is text -// label and value is localization. -func readLocales(fn string) (loc locales, err error) { - b, err := os.ReadFile(fn) +// readLocales reads file by name and returns a map, where the key is a text +// label and the value is a localization. +func readLocales(name string) (loc locales, err error) { + b, err := os.ReadFile(name) if err != nil { // Don't wrap the error since it's informative enough as is. return nil, err @@ -163,46 +155,44 @@ func readLocales(fn string) (loc locales, err error) { loc = make(locales) err = json.Unmarshal(b, &loc) if err != nil { - err = fmt.Errorf("unmarshalling %q: %w", fn, err) - - return nil, err + return nil, fmt.Errorf("unmarshalling %q: %w", name, err) } return loc, nil } -// summary prints summary for translations. -// -// TODO(e.burkov): Consider making it a method of [twoskyClient] and -// calculating summary for all configurations. -func summary(langs languages) (err error) { - basePath := filepath.Join(localesDirHome, defaultBaseFile) +// summary prints a summary for translations. +func (c *twoskyClient) summary() (err error) { + defer func() { err = errors.Annotate(err, "summary: %w") }() + + fmt.Printf("Summary for %s:\n\n", c.projectID) + + basePath := filepath.Join(c.localesDir, defaultBaseFile) baseLoc, err := readLocales(basePath) if err != nil { - return fmt.Errorf("summary: %w", err) + return fmt.Errorf("reading base locales: %w", err) } size := float64(len(baseLoc)) - for _, lang := range slices.Sorted(maps.Keys(langs)) { - name := filepath.Join(localesDirHome, string(lang)+".json") + for _, lang := range c.langs { + name := filepath.Join(c.localesDir, string(lang)+".json") if name == basePath { + // Skip base lang. continue } var loc locales loc, err = readLocales(name) if err != nil { - return fmt.Errorf("summary: reading locales: %w", err) + return fmt.Errorf("reading locales for %s: %w", lang, err) } f := float64(len(loc)) * 100 / size blocker := "" - // N is small enough to not raise performance questions. - ok := slices.Contains(blockerLangCodes, lang) - if ok { + if slices.Contains(blockerLangCodes, lang) { blocker = " (blocker)" } @@ -212,32 +202,39 @@ func summary(langs languages) (err error) { return nil } -// unused prints unused text labels. -// -// TODO(e.burkov): Consider making it a method of [twoskyClient] and searching -// unused strings for all configurations. -func unused(ctx context.Context, l *slog.Logger, basePath string) (err error) { +// unused prints unused text labels. l must not be nil. +func (c *twoskyClient) unused(ctx context.Context, l *slog.Logger) (err error) { defer func() { err = errors.Annotate(err, "unused: %w") }() - baseLoc, err := readLocales(basePath) + fmt.Printf("Unused for %s:\n\n", c.projectID) + fmt.Println() + + basePath := filepath.Join(c.localesDir, defaultBaseFile) + baseLocales, err := readLocales(basePath) if err != nil { - return err + return fmt.Errorf("reading locales: %w", err) } - locDir := filepath.Clean(localesDirHome) - js, err := findJS(ctx, l, locDir) + srcFiles, err := c.findSourceFiles(ctx, l) if err != nil { - return err + return fmt.Errorf("collecting js files: %w", err) } - return findUnused(js, baseLoc) + err = findUnused(srcFiles, baseLocales) + if err != nil { + return fmt.Errorf("finding unused files: %w", err) + } + + return nil } -// findJS returns list of JavaScript and JSON files or error. -func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []string, err error) { - walkFn := func(name string, _ os.FileInfo, err error) error { - if err != nil { - l.WarnContext(ctx, "accessing a path", slogutil.KeyError, err) +// findSourceFiles returns list of source files. l must not be nil. +func (c *twoskyClient) findSourceFiles(ctx context.Context, l *slog.Logger) (fileNames []string, err error) { + locDir := filepath.Clean(c.localesDir) + + walkFn := func(name string, _ os.FileInfo, pErr error) (err error) { + if pErr != nil { + l.WarnContext(ctx, "accessing a path", slogutil.KeyError, pErr) return nil } @@ -247,16 +244,16 @@ func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []str } ext := filepath.Ext(name) - if ext == ".js" || ext == ".json" { + if ext == ".js" || ext == ".json" || ext == ".ts" || ext == ".tsx" { fileNames = append(fileNames, name) } return nil } - err = filepath.Walk(srcDir, walkFn) + err = filepath.Walk(c.sourcesDir, walkFn) if err != nil { - return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err) + return nil, fmt.Errorf("walking %q: %w", c.sourcesDir, err) } return fileNames, nil @@ -278,7 +275,7 @@ func findUnused(fileNames []string, loc locales) (err error) { var buf []byte buf, err = os.ReadFile(fn) if err != nil { - return fmt.Errorf("finding unused: %w", err) + return fmt.Errorf("reading file: %w", err) } for k := range loc { @@ -296,18 +293,19 @@ func findUnused(fileNames []string, loc locales) (err error) { } // autoAdd adds locales with additions to the git and restores locales with -// deletions. -func autoAdd(ctx context.Context, l *slog.Logger, basePath string) (err error) { +// deletions. l must not be nil. +func (c *twoskyClient) autoAdd(ctx context.Context, l *slog.Logger) (err error) { defer func() { err = errors.Annotate(err, "auto add: %w") }() cmdCons := executil.SystemCommandConstructor{} - adds, dels, err := changedLocales(ctx, l, cmdCons) + adds, dels, err := c.changedLocales(ctx, l, cmdCons) if err != nil { // Don't wrap the error since it's informative enough as is. return err } + basePath := filepath.Join(c.localesDir, defaultBaseFile) if slices.Contains(dels, basePath) { return errors.Error("base locale contains deletions") } @@ -370,15 +368,15 @@ func handleDels( // changedLocales returns cleaned paths of locales with changes or error. adds // is the list of locales with only additions. dels is the list of locales -// with only deletions. -func changedLocales( +// with only deletions. l and cmdCons must not be nil. +func (c *twoskyClient) changedLocales( ctx context.Context, l *slog.Logger, cmdCons executil.CommandConstructor, ) (adds, dels []string, err error) { defer func() { err = errors.Annotate(err, "getting changes: %w") }() - gitArgs := []string{"diff", "--numstat", localesDirHome} + gitArgs := []string{"diff", "--numstat", c.localesDir} l.DebugContext(ctx, "executing", "cmd", gitCmd, "args", gitArgs) // TODO(s.chzhen): Consider streaming the output if needed. Using diff --git a/scripts/translations/twosky.go b/scripts/translations/twosky.go index e4229277256..32244e75fc6 100644 --- a/scripts/translations/twosky.go +++ b/scripts/translations/twosky.go @@ -13,30 +13,20 @@ import ( "github.com/AdguardTeam/golibs/validate" ) -// Constants for mapping the twosky configurations. -// -// Keep in sync with the .twosky.json file. -const ( - // twoskyProjectIdxHome is the index of the Home project in the localization - // configuration. - twoskyProjectIdxHome = iota - - // twoskyProjectIdxServices is the index of the Services project in the - // localization configuration. - twoskyProjectIdxServices - - // twoskyProjectCount is the number of projects in the localization - // configuration. - twoskyProjectCount -) +// metadata represents additional information for a single project. +type metadata struct { + LocalesDir string `json:"locales_dir"` + SourcesDir string `json:"sources_dir"` +} // twoskyConfig is the configuration structure for localization of a single // project. type twoskyConfig struct { Languages languages `json:"languages"` ProjectID string `json:"project_id"` - BaseLangcode langCode `json:"base_locale"` + BaseLangCode langCode `json:"base_locale"` LocalizableFiles []string `json:"localizable_files"` + Metadata metadata `json:"metadata"` } // type check @@ -50,8 +40,11 @@ func (t *twoskyConfig) Validate() (err error) { errs := []error{ validate.NotEmpty("project_id", t.ProjectID), - validate.NotEmpty("base_locale", t.BaseLangcode), + validate.NotEmpty("base_locale", t.BaseLangCode), validate.NotEmptySlice("localizable_files", t.LocalizableFiles), + validate.NotEmpty("metadata", t.Metadata), + validate.NotEmpty("metadata.locales_dir", t.Metadata.LocalesDir), + validate.NotEmpty("metadata.sources_dir", t.Metadata.SourcesDir), } if len(t.Languages) == 0 { @@ -68,48 +61,63 @@ func (t *twoskyConfig) Validate() (err error) { return errors.Join(errs...) } -// readTwoskyConfig returns twosky configuration. -func readTwoskyConfig() (home, services *twoskyConfig, err error) { +// newTwoskyConfig returns new twosky configuration. +func newTwoskyConfig() (conf *twoskyConfig, err error) { defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }() b, err := os.ReadFile(twoskyConfFile) if err != nil { // Don't wrap the error since it's informative enough as is. - return nil, nil, err + return nil, err } - var tsc []*twoskyConfig - err = json.Unmarshal(b, &tsc) + var confs []*twoskyConfig + err = json.Unmarshal(b, &confs) if err != nil { - return nil, nil, fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err) + return nil, fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err) } - err = validate.Equal("projects count", len(tsc), twoskyProjectCount) + err = validate.NotEmptySlice("projects", confs) if err != nil { - return nil, nil, err + // Don't wrap the error since it's informative enough as is. + return nil, err } - err = errors.Join(validate.AppendSlice(nil, "projects", tsc)...) + err = errors.Join(validate.AppendSlice(nil, "projects", confs)...) if err != nil { - return nil, nil, err + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + projectID := cmp.Or(os.Getenv("TWOSKY_PROJECT_ID"), defaultProjectID) + for _, c := range confs { + if c.ProjectID == projectID { + return c, nil + } } - return tsc[twoskyProjectIdxHome], tsc[twoskyProjectIdxServices], nil + return nil, fmt.Errorf("project %q not found in %s", projectID, twoskyConfFile) } -// twoskyClient is the twosky client with methods for download and upload -// translations. +// twoskyClient is the client for the twosky translation service. type twoskyClient struct { // uri is the base URL. uri *url.URL + // baseLang is the base language code. + baseLang langCode + + // localesDir is the path to the directory with locale files. + localesDir string + // projectID is the name of the project. projectID string - // baseLang is the base language code. - baseLang langCode + // sourcesDir is the path to directory with source files in there the + // localizations are used. + sourcesDir string - // langs is the list of codes of languages to download. + // langs is the list of codes of languages. langs []langCode // localizableFiles are the files to localize. @@ -117,7 +125,7 @@ type twoskyClient struct { } // newTwoskyClient reads values from environment variables or defaults, -// validates them, and returns the twosky client. +// validates them, and returns the twosky client. conf must be valid. func newTwoskyClient(conf *twoskyConfig) (cli *twoskyClient, err error) { defer func() { err = errors.Annotate(err, "filling config: %w") }() @@ -134,7 +142,7 @@ func newTwoskyClient(conf *twoskyConfig) (cli *twoskyClient, err error) { projectID = envProjectID } - baseLang := conf.BaseLangcode + baseLang := conf.BaseLangCode uLangStr := os.Getenv("UPLOAD_LANGUAGE") if uLangStr != "" { baseLang = langCode(uLangStr) @@ -157,8 +165,10 @@ func newTwoskyClient(conf *twoskyConfig) (cli *twoskyClient, err error) { return &twoskyClient{ uri: uri, - projectID: projectID, baseLang: baseLang, + localesDir: conf.Metadata.LocalesDir, + projectID: projectID, + sourcesDir: conf.Metadata.SourcesDir, langs: langs, localizableFiles: conf.LocalizableFiles, }, nil diff --git a/scripts/translations/upload.go b/scripts/translations/upload.go index 794cc717231..efd084623b1 100644 --- a/scripts/translations/upload.go +++ b/scripts/translations/upload.go @@ -22,7 +22,7 @@ func (c *twoskyClient) upload() (err error) { defer func() { err = errors.Annotate(err, "upload: %w") }() uploadURI := c.uri.JoinPath("upload") - basePath := filepath.Join(localesDirHome, defaultBaseFile) + basePath := filepath.Join(c.localesDir, defaultBaseFile) formData := map[string]string{ "format": "json",