[Feat] Translates + translate tools (#260)

* autotranslate python script

* 8/52 translated

* autotranslate finished

* updated autotranslate script

* fixes

* updated tool

* once more updated tool

* more translates

* more translates

* fixes

Tools/ss14_ru/__init__.py

@@ -0,0 +1 @@
+from fluentformatter import FluentFile, FluentFormatter

Tools/ss14_ru/file.py

@@ -0,0 +1,91 @@
+import typing
+
+from fluent.syntax import ast
+from yamlmodels import YAMLElements
+import os
+
+
+class File:
+    def __init__(self, full_path):
+        self.full_path = full_path
+
+    def read_data(self):
+        file = open(self.full_path, 'r', encoding='utf8')
+        # replace необходим для того, чтобы 1-е сообщение не считалось ast.Junk
+        file_data = file.read().replace('', '')
+        file.close()
+
+        return file_data
+
+    def save_data(self, file_data: typing.AnyStr):
+        os.makedirs(os.path.dirname(self.full_path), exist_ok=True)
+        file = open(self.full_path, 'w', encoding='utf8')
+        file.write(file_data)
+        file.close()
+
+    def get_relative_path(self, base_path):
+        return os.path.relpath(self.full_path, base_path)
+
+    def get_relative_path_without_extension(self, base_path):
+        return self.get_relative_path(base_path).split('.', maxsplit=1)[0]
+
+    def get_relative_parent_dir(self, base_path):
+        return os.path.relpath(self.get_parent_dir(), base_path)
+
+    def get_parent_dir(self):
+        return os.path.dirname(self.full_path)
+
+    def get_name(self):
+        return os.path.basename(self.full_path).split('.')[0]
+
+
+class FluentFile(File):
+    def __init__(self, full_path):
+        super().__init__(full_path)
+        self.full_path = full_path
+
+    def parse_data(self, file_data: typing.AnyStr):
+        from fluent.syntax import FluentParser
+
+        return FluentParser().parse(file_data)
+
+    def serialize_data(self, parsed_file_data: ast.Resource):
+        from fluent.syntax import FluentSerializer
+
+        return FluentSerializer(with_junk=True).serialize(parsed_file_data)
+
+    def read_serialized_data(self):
+        return self.serialize_data(self.parse_data(self.read_data()))
+
+    def read_parsed_data(self):
+        return self.parse_data(self.read_data())
+
+
+class YAMLFluentFileAdapter(File):
+    def __init__(self, full_path):
+        super().__init__(full_path)
+        self.full_path = full_path
+
+    # def create_fluent_from_yaml_elements(self, yaml_elements):
+
+
+
+class YAMLFile(File):
+    def __init__(self, full_path):
+        super().__init__(full_path)
+
+    def parse_data(self, file_data: typing.AnyStr):
+        import yaml
+
+        return yaml.load(file_data, Loader=yaml.BaseLoader)
+
+    def get_elements(self, parsed_data):
+
+        if isinstance(parsed_data, list):
+            elements = YAMLElements(parsed_data).elements
+            # элемент может быть None, если имеет неизвестный тип
+            exist_elements = list(filter(lambda el: el, elements))
+
+            return exist_elements
+
+        return []

Tools/ss14_ru/fluentast.py

@@ -0,0 +1,176 @@
+import typing
+
+from fluent.syntax import ast, FluentParser, FluentSerializer
+from lokalisemodels import LokaliseKey
+from pydash import py_
+
+
+class FluentAstAbstract:
+    element = None
+    @classmethod
+    def get_id_name(cls, element):
+        if isinstance(element, ast.Junk):
+            return FluentAstJunk(element).get_id_name()
+        elif isinstance(element, ast.Message):
+            return FluentAstMessage(element).get_id_name()
+        else:
+            return None
+
+    @classmethod
+    def create_element(cls, element):
+        if isinstance(element, ast.Junk):
+            cls.element = FluentAstJunk(element)
+            return cls.element
+        elif isinstance(element, ast.Message):
+            cls.element = FluentAstMessage(element)
+            return cls.element
+        else:
+            return None
+
+
+class FluentAstMessage:
+    def __init__(self, message: ast.Message):
+        self.message = message
+        self.element = message
+
+    def get_id_name(self):
+        return self.message.id.name
+
+
+class FluentAstAttribute:
+    def __init__(self, id, value, parent_key = None):
+        self.id = id
+        self.value = value
+        self.parent_key = parent_key
+
+
+class FluentAstAttributeFactory:
+    @classmethod
+    def from_yaml_element(cls, element):
+        attrs = []
+        if element.description:
+            attrs.append(FluentAstAttribute('desc', element.description))
+
+        if element.suffix:
+            attrs.append(FluentAstAttribute('suffix', element.suffix))
+
+        if not len(attrs):
+            return None
+
+        return attrs
+
+
+class FluentAstJunk:
+    def __init__(self, junk: ast.Junk):
+        self.junk = junk
+        self.element = junk
+
+    def get_id_name(self):
+        return self.junk.content.split('=')[0].strip()
+
+
+class FluentSerializedMessage:
+    @classmethod
+    def from_yaml_element(cls, id, value, attributes, parent_id = None, raw_key = False):
+        if not value and not id and not parent_id:
+            return None
+
+        if not attributes:
+            attributes = []
+
+        if len(list(filter(lambda attr: attr.id == 'desc', attributes))) == 0:
+            if parent_id:
+                attributes.append(FluentAstAttribute('desc', '{ ' + FluentSerializedMessage.get_key(parent_id) + '.desc' + ' }'));
+            else:
+                attributes.append(FluentAstAttribute('desc', '{ "" }'))
+
+        if len(list(filter(lambda attr: attr.id == 'suffix', attributes))) == 0:
+            attributes.append(FluentAstAttribute('suffix', '{ "" }'))
+
+        message = f'{cls.get_key(id, raw_key)} = {cls.get_value(value, parent_id)}\n'
+
+        if attributes and len(attributes):
+            full_message = message
+
+            for attr in attributes:
+                fluent_newlines = attr.value.replace("\n", "\n        ");
+                full_message = cls.add_attr(full_message, attr.id, fluent_newlines, raw_key=raw_key)
+
+            desc_attr = py_.find(attributes, lambda a: a.id == 'desc')
+            if not desc_attr and parent_id:
+                full_message = cls.add_attr(full_message, 'desc', '{ ' + FluentSerializedMessage.get_key(parent_id) + '.desc' + ' }')
+
+            return full_message
+
+        return cls.to_serialized_message(message)
+
+    @classmethod
+    def from_lokalise_keys(cls, keys: typing.List[LokaliseKey]):
+        attributes_keys = list(filter(lambda k: k.is_attr, keys))
+        attributes = list(map(lambda k: FluentAstAttribute(id='.{name}'.format(name=k.get_key_last_name(k.key_name)),
+                                                           value=FluentSerializedMessage.get_attr(k, k.get_key_last_name(k.key_name)), parent_key=k.get_parent_key()),
+                              attributes_keys))
+        attributes_group = py_.group_by(attributes, 'parent_key')
+
+        serialized_message = ''
+        for key in keys:
+            if key.is_attr:
+                continue
+            key_name = key.get_key_last_name(key.key_name)
+            key_value = key.get_translation('ru').data['translation']
+            key_attributes = []
+
+            if len(attributes_group):
+                k = f'{key.get_key_base_name(key.key_name)}.{key_name}'
+                key_attributes = attributes_group[k] if k in attributes_group else []
+
+            message = key.serialize_message()
+            full_message = cls.from_yaml_element(key_name, key_value, key_attributes, key.get_parent_key(), True)
+
+            if full_message:
+                serialized_message = serialized_message + '\n' + full_message
+            elif message:
+                serialized_message = serialized_message + '\n' + message
+            else:
+                raise Exception('Что-то пошло не так')
+
+        return serialized_message
+
+    @staticmethod
+    def get_attr(k, name, parent_id = None):
+        if parent_id:
+            return "{ " + parent_id + f'.{name}' + " }"
+        else:
+            return k.get_translation('ru').data['translation']
+
+
+    @staticmethod
+    def to_serialized_message(string_message):
+        if not string_message:
+            return None
+
+        ast_message = FluentParser().parse(string_message)
+        serialized = FluentSerializer(with_junk=True).serialize(ast_message)
+
+        return serialized if serialized else ''
+
+    @staticmethod
+    def add_attr(message_str, attr_key, attr_value, raw_key = False):
+        prefix = '' if raw_key else '.'
+        return f'{message_str}\n  {prefix}{attr_key} = {attr_value}'
+
+    @staticmethod
+    def get_value(value, parent_id):
+        if value:
+            return value
+        elif parent_id:
+            return '{ ' + FluentSerializedMessage.get_key(parent_id) + ' }'
+        else:
+            return '{ "" }'
+
+    @staticmethod
+    def get_key(id, raw = False):
+        if raw:
+            return f'{id}'
+        else:
+            return f'ent-{id}'

Tools/ss14_ru/fluentastcomparer.py

@@ -0,0 +1,100 @@
+from fluent.syntax import ast
+from fluentast import FluentAstAbstract
+from pydash import py_
+
+
+class FluentAstComparer:
+    def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource):
+        self.sourse_parsed = sourse_parsed
+        self.target_parsed = target_parsed
+        self.source_elements = list(
+            filter(lambda el: el, list(map(lambda e: FluentAstAbstract.create_element(e), sourse_parsed.body))))
+        self.target_elements = list(
+            filter(lambda el: el, list(map(lambda e: FluentAstAbstract.create_element(e), target_parsed.body))))
+
+    # Возвращает полностью эквивалентные сообщения (не считая span)
+    def get_equal_elements(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span'])
+
+        return py_.intersection_with(self.source_elements, self.target_elements, comparator=comparator)
+
+    # Возвращает полностью неэквивалентные сообщения (не считая span)
+    def get_not_equal_elements(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span'])
+        diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения с эквивалентными именами ключей
+    def get_equal_id_names(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        eq = py_.intersection_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return eq
+
+    # Возвращает сообщения с неэквивалентными именами ключей
+    def get_not_equal_id_names(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения target, существующие в source
+    def get_exist_id_names(self, source, target):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        eq = py_.intersection_with(source, target, comparator=comparator)
+
+        return eq
+
+    # Возвращает сообщения target, существующие в source
+    def get_not_exist_id_names(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        diff = py_.difference_with(self.target_elements, self.source_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения с эквивалентным значением и атрибутами
+    def get_equal_values_with_attrs(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment'])
+        eq = py_.intersection_with(self.target_elements, self.source_elements, comparator=comparator)
+
+        return eq
+
+    # Возвращает сообщения из source с неэквивалентным значением и атрибутами
+    def get_not_equal_values_with_attrs(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment'])
+        diff = py_.difference_with(self.source_elements, self.target_elements,
+                                   comparator=lambda a, b: a.element.equals(b.element,
+                                                                            ignored_fields=['span', 'id', 'comment']))
+
+        return diff
+
+    # Возвращает сообщения из source, существующие в target и source, с неэквивалентным значением и атрибутами
+    def get_not_equal_exist_values_with_attrs(self):
+        diff = py_.difference_with(self.source_elements, self.target_elements,
+                                   comparator=lambda a, b: a.element.equals(b.element,
+                                                                            ignored_fields=['span', 'id', 'comment']))
+        ex = self.get_exist_id_names(self.source_elements, self.target_elements)
+        exist = py_.intersection(diff, ex)
+
+        return exist
+
+        # Возвращает сообщения из target с неэквивалентным значением и атрибутами
+
+    def get_target_not_equal_values_with_attrs(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment'])
+        diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения, существующие в target и source, с неэквивалентным значением и атрибутами
+    def get_target_not_equal_exist_values_with_attrs(self):
+        diff = py_.difference_with(self.target_elements, self.source_elements,
+                                   comparator=lambda a, b: a.element.equals(b.element,
+                                                                            ignored_fields=['span', 'id', 'comment']))
+        exist = py_.intersection(diff, self.get_exist_id_names(self.target_elements, self.source_elements))
+
+        return exist
+
+    def find_message_by_id_name(self, id_name, list):
+        return py_.find(list, lambda el: el.get_id_name() == id_name)

Tools/ss14_ru/fluentastmanager.py

@@ -0,0 +1,25 @@
+from fluent.syntax import ast
+from fluentast import FluentAstAbstract
+
+
+class FluentAstManager:
+    def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource):
+        self.sourse_parsed = sourse_parsed
+        self.target_parsed = target_parsed
+        self.source_elements = list(map(lambda e: FluentAstAbstract.create_element(e), sourse_parsed.body))
+        self.target_elements = list(map(lambda e: FluentAstAbstract.create_element(e), target_parsed.body))
+
+    def update_by_index(self, index, update_element: ast.Message):
+        source_element = None
+
+        try:
+            source_element = self.sourse_parsed.body[index]
+        except:
+            raise Exception(f'Нет элемента с индексом {index}')
+
+        if not source_element:
+            raise Exception(f'Элемен с индексом {index} не существует')
+
+        self.sourse_parsed.body[index] = update_element
+
+        return self.sourse_parsed

Tools/ss14_ru/fluentformatter.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# Форматтер, приводящий fluent-файлы (.ftl) в соответствие стайлгайду
+# path - путь к папке, содержащий форматируемые файлы. Для форматирования всего проекта, необходимо заменить значение на root_dir_path
+import typing
+
+from file import FluentFile
+from project import Project
+from fluent.syntax import ast, FluentParser, FluentSerializer
+
+
+######################################### Class defifitions ############################################################
+
+class FluentFormatter:
+    @classmethod
+    def format(cls, fluent_files: typing.List[FluentFile]):
+        for file in fluent_files:
+            file_data = file.read_data()
+            parsed_file_data = file.parse_data(file_data)
+            serialized_file_data = file.serialize_data(parsed_file_data)
+            file.save_data(serialized_file_data)
+
+    @classmethod
+    def format_serialized_file_data(cls, file_data: typing.AnyStr):
+        parsed_data = FluentParser().parse(file_data)
+
+        return FluentSerializer(with_junk=True).serialize(parsed_data)
+
+
+
+######################################## Var definitions ###############################################################
+project = Project()
+fluent_files = project.get_fluent_files_by_dir(project.ru_locale_dir_path)
+
+########################################################################################################################
+
+FluentFormatter.format(fluent_files)

Tools/ss14_ru/keyfinder.py

@@ -0,0 +1,220 @@
+import typing
+import logging
+
+from pydash import py_
+
+from file import FluentFile
+from fluentast import FluentAstAbstract
+from fluentformatter import FluentFormatter
+from project import Project
+from fluent.syntax import ast, FluentParser, FluentSerializer
+
+
+# Осуществляет актуализацию ключей. Находит файлы английского перевода, проверяет: есть ли русскоязычная пара
+# Если нет - создаёт файл с копией переводов из англоязычного
+# Далее, пофайлово проверяются ключи. Если в английском файле больше ключей - создает недостающие в русском, с английской копией перевода
+# Отмечает русские файлы, в которых есть те ключи, что нет в аналогичных английских
+# Отмечает русские файлы, у которых нет англоязычной пары
+
+######################################### Class defifitions ############################################################
+class RelativeFile:
+    def __init__(self, file: FluentFile, locale: typing.AnyStr, relative_path_from_locale: typing.AnyStr):
+        self.file = file
+        self.locale = locale
+        self.relative_path_from_locale = relative_path_from_locale
+
+
+class FilesFinder:
+    def __init__(self, project: Project):
+        self.project: Project = project
+        self.created_files: typing.List[FluentFile] = []
+
+    def get_relative_path_dict(self, file: FluentFile, locale):
+        if locale == 'ru-RU':
+            return RelativeFile(file=file, locale=locale,
+                                relative_path_from_locale=file.get_relative_path(self.project.ru_locale_dir_path))
+        elif locale == 'en-US':
+            return RelativeFile(file=file, locale=locale,
+                                relative_path_from_locale=file.get_relative_path(self.project.en_locale_dir_path))
+        else:
+            raise Exception(f'Локаль {locale} не поддерживается')
+
+    def get_file_pair(self, en_file: FluentFile) -> typing.Tuple[FluentFile, FluentFile]:
+        ru_file_path = en_file.full_path.replace('en-US', 'ru-RU')
+        ru_file = FluentFile(ru_file_path)
+
+        return en_file, ru_file
+
+    def execute(self):
+        self.created_files = []
+        groups = self.get_files_pars()
+        keys_without_pair = list(filter(lambda g: len(groups[g]) < 2, groups))
+
+        for key_without_pair in keys_without_pair:
+            relative_file: RelativeFile = groups.get(key_without_pair)[0]
+
+            if relative_file.locale == 'en-US':
+                ru_file = self.create_ru_analog(relative_file)
+                self.created_files.append(ru_file)
+            elif relative_file.locale == 'ru-RU':
+                is_engine_files = "robust-toolbox" in (relative_file.file.full_path)
+                is_white_files = "white" in (relative_file.file.full_path)
+                if not is_engine_files and not is_white_files:
+                    self.warn_en_analog_not_exist(relative_file)
+            else:
+                raise Exception(f'Файл {relative_file.file.full_path} имеет неизвестную локаль "{relative_file.locale}"')
+
+        return self.created_files
+
+    def get_files_pars(self):
+        en_fluent_files = self.project.get_fluent_files_by_dir(project.en_locale_dir_path)
+        ru_fluent_files = self.project.get_fluent_files_by_dir(project.ru_locale_dir_path)
+
+        en_fluent_relative_files = list(map(lambda f: self.get_relative_path_dict(f, 'en-US'), en_fluent_files))
+        ru_fluent_relative_files = list(map(lambda f: self.get_relative_path_dict(f, 'ru-RU'), ru_fluent_files))
+        relative_files = py_.flatten_depth(py_.concat(en_fluent_relative_files, ru_fluent_relative_files), depth=1)
+
+        return py_.group_by(relative_files, 'relative_path_from_locale')
+
+    def create_ru_analog(self, en_relative_file: RelativeFile) -> FluentFile:
+        en_file: FluentFile = en_relative_file.file
+        en_file_data = en_file.read_data()
+        ru_file_path = en_file.full_path.replace('en-US', 'ru-RU')
+        ru_file = FluentFile(ru_file_path)
+        ru_file.save_data(en_file_data)
+
+        logging.info(f'Создан файл {ru_file_path} с переводами из английского файла')
+
+        return ru_file
+
+    def warn_en_analog_not_exist(self, ru_relative_file: RelativeFile):
+        file: FluentFile = ru_relative_file.file
+        en_file_path = file.full_path.replace('ru-RU', 'en-US')
+
+        logging.warning(f'Файл {file.full_path} не имеет английского аналога по пути {en_file_path}')
+
+
+class KeyFinder:
+    def __init__(self, files_dict):
+        self.files_dict = files_dict
+        self.changed_files: typing.List[FluentFile] = []
+
+    def execute(self) -> typing.List[FluentFile]:
+        self.changed_files = []
+        for pair in self.files_dict:
+            ru_relative_file = py_.find(self.files_dict[pair], {'locale': 'ru-RU'})
+            en_relative_file = py_.find(self.files_dict[pair], {'locale': 'en-US'})
+
+            if not en_relative_file or not ru_relative_file:
+                continue
+
+            ru_file: FluentFile = ru_relative_file.file
+            en_file: FluentFile = en_relative_file.file
+
+            self.compare_files(en_file, ru_file)
+
+        return self.changed_files
+
+
+    def compare_files(self, en_file, ru_file):
+        ru_file_parsed: ast.Resource = ru_file.parse_data(ru_file.read_data())
+        en_file_parsed: ast.Resource = en_file.parse_data(en_file.read_data())
+
+        self.write_to_ru_files(ru_file, ru_file_parsed, en_file_parsed)
+        self.log_not_exist_en_files(en_file, ru_file_parsed, en_file_parsed)
+
+
+    def write_to_ru_files(self, ru_file, ru_file_parsed, en_file_parsed):
+        for idx, en_message in enumerate(en_file_parsed.body):
+            if isinstance(en_message, ast.ResourceComment) or isinstance(en_message, ast.GroupComment) or isinstance(en_message, ast.Comment):
+                continue
+
+            ru_message_analog_idx = py_.find_index(ru_file_parsed.body, lambda ru_message: self.find_duplicate_message_id_name(ru_message, en_message))
+            have_changes = False
+
+            # Attributes
+            if getattr(en_message, 'attributes', None) and ru_message_analog_idx != -1:
+                if not ru_file_parsed.body[ru_message_analog_idx].attributes:
+                    ru_file_parsed.body[ru_message_analog_idx].attributes = en_message.attributes
+                    have_changes = True
+                else:
+                    for en_attr in en_message.attributes:
+                        ru_attr_analog = py_.find(ru_file_parsed.body[ru_message_analog_idx].attributes, lambda ru_attr: ru_attr.id.name == en_attr.id.name)
+                        if not ru_attr_analog:
+                            ru_file_parsed.body[ru_message_analog_idx].attributes.append(en_attr)
+                            have_changes = True
+
+            # New elements
+            if ru_message_analog_idx == -1:
+                ru_file_body = ru_file_parsed.body
+                if (len(ru_file_body) >= idx + 1):
+                    ru_file_parsed = self.append_message(ru_file_parsed, en_message, idx)
+                else:
+                    ru_file_parsed = self.push_message(ru_file_parsed, en_message)
+                have_changes = True
+
+            if have_changes:
+                serialized = serializer.serialize(ru_file_parsed)
+                self.save_and_log_file(ru_file, serialized, en_message)
+
+    def log_not_exist_en_files(self, en_file, ru_file_parsed, en_file_parsed):
+        for idx, ru_message in enumerate(ru_file_parsed.body):
+            if isinstance(ru_message, ast.ResourceComment) or isinstance(ru_message, ast.GroupComment) or isinstance(ru_message, ast.Comment):
+                continue
+
+            en_message_analog = py_.find(en_file_parsed.body, lambda en_message: self.find_duplicate_message_id_name(ru_message, en_message))
+
+            if not en_message_analog:
+                logging.warning(f'Ключ "{FluentAstAbstract.get_id_name(ru_message)}" не имеет английского аналога по пути {en_file.full_path}"')
+
+    def append_message(self, ru_file_parsed, en_message, en_message_idx):
+        ru_message_part_1 = ru_file_parsed.body[0:en_message_idx]
+        ru_message_part_middle = [en_message]
+        ru_message_part_2 = ru_file_parsed.body[en_message_idx:]
+        new_body = py_.flatten_depth([ru_message_part_1, ru_message_part_middle, ru_message_part_2], depth=1)
+        ru_file_parsed.body = new_body
+
+        return ru_file_parsed
+
+    def push_message(self,  ru_file_parsed, en_message):
+        ru_file_parsed.body.append(en_message)
+        return ru_file_parsed
+
+    def save_and_log_file(self, file, file_data, message):
+        file.save_data(file_data)
+        logging.info(f'В файл {file.full_path} добавлен ключ "{FluentAstAbstract.get_id_name(message)}"')
+        self.changed_files.append(file)
+
+    def find_duplicate_message_id_name(self, ru_message, en_message):
+        ru_element_id_name = FluentAstAbstract.get_id_name(ru_message)
+        en_element_id_name = FluentAstAbstract.get_id_name(en_message)
+
+        if not ru_element_id_name or not en_element_id_name:
+            return False
+
+        if ru_element_id_name == en_element_id_name:
+            return ru_message
+        else:
+            return None
+
+######################################## Var definitions ###############################################################
+
+logging.basicConfig(level = logging.INFO)
+project = Project()
+parser = FluentParser()
+serializer = FluentSerializer(with_junk=True)
+files_finder = FilesFinder(project)
+key_finder = KeyFinder(files_finder.get_files_pars())
+
+########################################################################################################################
+
+print('Проверка актуальности файлов ...')
+created_files = files_finder.execute()
+if len(created_files):
+    print('Форматирование созданных файлов ...')
+    FluentFormatter.format(created_files)
+print('Проверка актуальности ключей ...')
+changed_files = key_finder.execute()
+if len(changed_files):
+    print('Форматирование изменённых файлов ...')
+    FluentFormatter.format(changed_files)

Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py

@@ -0,0 +1,48 @@
+from fluent.syntax import ast
+
+from fluentast import FluentAstMessage
+from fluentastcomparer import FluentAstComparer
+from fluentastmanager import FluentAstManager
+
+
+class LokaliseFluentAstComparerManager:
+    def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource):
+        self.sourse_parsed = sourse_parsed
+        self.target_parsed = target_parsed
+        self.comparer = FluentAstComparer(sourse_parsed, target_parsed)
+        self.ast_manager = FluentAstManager(sourse_parsed, target_parsed)
+
+    def for_update(self):
+        for_update = self.comparer.get_not_equal_exist_values_with_attrs()
+
+        if not len(for_update):
+            return []
+
+        return for_update
+
+    def update(self, for_update):
+        for update in for_update:
+            idx = self.comparer.sourse_parsed.body.index(update.element)
+            update_mess: FluentAstMessage = self.comparer.find_message_by_id_name(update.get_id_name(),
+                                                                                  self.comparer.target_elements)
+            self.ast_manager.update_by_index(idx, update_mess.element)
+
+        return self.ast_manager.sourse_parsed
+
+    def for_delete(self):
+        for_delete = self.comparer.get_not_exist_id_names()
+
+        if len(for_delete):
+            keys = list(map(lambda el: el.get_id_name(), for_delete))
+            print(f'Следующие ключи есть в lokalise, но нет в файле. Возможно, их нужно удалить из lokalise: {keys}')
+
+        return for_delete
+
+    def for_create(self):
+        for_create = self.comparer.get_not_equal_id_names()
+
+        if len(for_create):
+            keys = list(map(lambda el: el.get_id_name(), for_create))
+            print(f'Следующих ключей файла нет в lokalise. Необходимо добавить: {keys}')
+
+        return for_create

Tools/ss14_ru/lokalise_project.py

@@ -0,0 +1,33 @@
+import lokalise
+import typing
+from lokalisemodels import LokaliseKey
+from pydash import py_
+
+class LokaliseProject:
+    def __init__(self, project_id, personal_token):
+        self.project_id = project_id
+        self.personal_token = personal_token
+        self.client = lokalise.Client(self.personal_token)
+
+    def get_all_keys(self) -> typing.List[LokaliseKey]:
+        page = 1
+        keys = self.get_keys(page=page)
+        keys_items: typing.List[lokalise.client.KeyModel] = []
+        general_count = 0
+
+        while (general_count < keys.total_count):
+            general_count = general_count + len(keys.items)
+            keys_items = py_.flatten_depth(py_.concat(keys_items, keys.items), depth=1)
+
+            if (general_count == keys.total_count):
+                break
+
+            next_page = page = page + 1
+            keys = self.get_keys(page=next_page)
+
+        sorted_list = py_.sort(keys_items, key=lambda item: item.translations_modified_at_timestamp, reverse=True)
+
+        return list(map(lambda k: LokaliseKey(k), sorted_list))
+
+    def get_keys(self, page):
+        return self.client.keys(self.project_id, {'page': page, 'limit': 5000, 'include_translations': 1})

Tools/ss14_ru/lokalisemodels.py

@@ -0,0 +1,64 @@
+import typing
+import os
+from pydash import py_
+from project import Project
+
+class LocalePath:
+    def __init__(self, relative_file_path):
+        self.ru = os.path.join(Project().ru_locale_dir_path, relative_file_path)
+        self.en = os.path.join(Project().en_locale_dir_path, relative_file_path)
+
+
+class LokaliseTranslation:
+    def __init__(self, data, key_name: typing.AnyStr):
+        self.key_name = key_name,
+        self.data = data
+
+class LokaliseKey:
+    def __init__(self, data):
+        self.data = data
+        self.key_name = self.data.key_name['web']
+        self.key_base_name = self.get_key_base_name(self.key_name)
+        self.is_attr = self.check_is_attr()
+
+    def get_file_path(self):
+
+        relative_dir_path = '{relative_file_path}.ftl'.format(
+            relative_file_path='/'.join(self.data.key_name['web'].split('.')[0].split('::')))
+
+        return LocalePath(relative_dir_path)
+
+    def get_key_base_name(self, key_name):
+        splitted_name = key_name.split('.')
+        return splitted_name[0]
+
+    def get_key_last_name(self, key_name):
+        splitted_name = key_name.split('.')
+        return py_.last(splitted_name)
+
+    def get_parent_key(self):
+        if self.is_attr:
+            splitted_name = self.key_name.split('.')[0:-1]
+            return '.'.join(splitted_name)
+        else:
+            return None
+
+    def check_is_attr(self):
+        return len(self.key_name.split('.')) > 2
+
+    def serialize(self):
+        if self.is_attr:
+            return self.serialize_attr()
+        else:
+            return self.serialize_message()
+
+
+
+    def serialize_attr(self):
+        return '.{name} = {value}'.format(name=self.get_key_last_name(self.key_name), value=self.get_translation('ru').data['translation'])
+
+    def serialize_message(self):
+        return '{name} = {value}'.format(name=self.get_key_last_name(self.key_name), value=self.get_translation('ru').data['translation'])
+
+    def get_translation(self, language_iso='ru'):
+        return list(map(lambda data: LokaliseTranslation(key_name=self.data.key_name['web'], data=data), py_.filter(self.data.translations, {'language_iso': language_iso})))[0]

Tools/ss14_ru/project.py

@@ -0,0 +1,31 @@
+import pathlib
+import os
+import glob
+from file import FluentFile
+
+class Project:
+    def __init__(self):
+        self.base_dir_path = pathlib.Path(os.path.abspath(os.curdir)).parent.parent.resolve()
+        self.resources_dir_path = os.path.join(self.base_dir_path, 'Resources')
+        self.locales_dir_path = os.path.join(self.resources_dir_path, 'Locale')
+        self.ru_locale_dir_path = os.path.join(self.locales_dir_path, 'ru-RU')
+        self.en_locale_dir_path = os.path.join(self.locales_dir_path, 'en-US')
+        self.prototypes_dir_path = os.path.join(self.resources_dir_path, "Prototypes")
+        self.en_locale_prototypes_dir_path = os.path.join(self.en_locale_dir_path, 'ss14-ru', 'prototypes')
+        self.ru_locale_prototypes_dir_path = os.path.join(self.ru_locale_dir_path, 'ss14-ru', 'prototypes')
+
+    def get_files_paths_by_dir(self, dir_path, files_extenstion):
+        return glob.glob(f'{dir_path}/**/*.{files_extenstion}', recursive=True)
+
+    def get_fluent_files_by_dir(self, dir_path):
+        files = []
+        files_paths_list = glob.glob(f'{dir_path}/**/*.ftl', recursive=True)
+
+        for file_path in files_paths_list:
+            try:
+                files.append(FluentFile(file_path))
+            except:
+                continue
+
+        return files
+

Tools/ss14_ru/translationsassembler.py

@@ -0,0 +1,67 @@
+import logging
+import typing
+
+from fluent.syntax import FluentParser, FluentSerializer
+from pydash import py_
+
+from file import FluentFile
+from fluentast import FluentSerializedMessage
+from lokalise_fluent_ast_comparer_manager import LokaliseFluentAstComparerManager
+from lokalise_project import LokaliseProject
+from lokalisemodels import LokaliseKey
+import os
+
+######################################### Class defifitions ############################################################
+
+# TODO непереведенные элементы приходят как { "" }. Необходимо сохранять английский перевод
+class TranslationsAssembler:
+    def __init__(self, items: typing.List[LokaliseKey]):
+        self.group = py_.group_by(items, 'key_base_name')
+        keys = list(self.group.keys())
+        self.sorted_keys = py_.sort_by(keys, lambda key: self.sort_by_translations_timestamp(self.group[key]),
+                                       reverse=True)
+
+    def execute(self):
+        for keys in self.group:
+            full_message = FluentSerializedMessage.from_lokalise_keys(self.group[keys])
+            parsed_message = FluentParser().parse(full_message)
+            ru_full_path = self.group[keys][0].get_file_path().ru
+            ru_file = FluentFile(ru_full_path)
+            try:
+                ru_file_parsed = ru_file.read_parsed_data()
+            except:
+                logging.error(f'Файла {ru_file.full_path} не существует')
+                continue
+
+            manager = LokaliseFluentAstComparerManager(sourse_parsed=ru_file_parsed, target_parsed=parsed_message)
+
+            for_update = manager.for_update()
+            for_create = manager.for_create()
+            for_delete = manager.for_delete()
+
+            if len(for_update):
+                updated_ru_file_parsed = manager.update(for_update)
+                updated_ru_file_serialized = FluentSerializer(with_junk=True).serialize(updated_ru_file_parsed)
+                ru_file.save_data(updated_ru_file_serialized)
+
+                updated_keys = list(map(lambda el: el.get_id_name(), for_update))
+                logging.info(f'Обновлены ключи: {updated_keys} в файле {ru_file.full_path}')
+
+    def sort_by_translations_timestamp(self, list):
+        sorted_list = py_.sort_by(list, 'data.translations_modified_at_timestamp', reverse=True)
+
+        return sorted_list[0].data.translations_modified_at_timestamp
+
+######################################## Var definitions ###############################################################
+
+logging.basicConfig(level=logging.INFO)
+lokalise_project_id = os.getenv('lokalise_project_id')
+lokalise_personal_token = os.getenv('lokalise_personal_token')
+lokalise_project = LokaliseProject(project_id=lokalise_project_id,
+                                   personal_token=lokalise_personal_token)
+all_keys: typing.List[LokaliseKey] = lokalise_project.get_all_keys()
+translations_assembler = TranslationsAssembler(all_keys)
+
+########################################################################################################################
+
+translations_assembler.execute()

Tools/ss14_ru/yamlextractor.py

@@ -0,0 +1,85 @@
+import os
+
+from fluent.syntax.parser import FluentParser
+from fluent.syntax.serializer import FluentSerializer
+
+from file import YAMLFile, FluentFile
+from fluentast import FluentSerializedMessage, FluentAstAttributeFactory
+from fluentformatter import FluentFormatter
+from project import Project
+import logging
+
+######################################### Class defifitions ############################################################
+class YAMLExtractor:
+    def __init__(self, yaml_files):
+        self.yaml_files = yaml_files
+
+    def execute(self):
+        for yaml_file in yaml_files:
+            yaml_elements = yaml_file.get_elements(yaml_file.parse_data(yaml_file.read_data()))
+
+            if not len(yaml_elements):
+                continue
+
+            fluent_file_serialized = self.get_serialized_fluent_from_yaml_elements(yaml_elements)
+
+            if not fluent_file_serialized:
+                continue
+
+            pretty_fluent_file_serialized = formatter.format_serialized_file_data(fluent_file_serialized)
+
+            relative_parent_dir = yaml_file.get_relative_parent_dir(project.prototypes_dir_path).lower()
+            file_name = yaml_file.get_name()
+
+            en_fluent_file_path = self.create_en_fluent_file(relative_parent_dir, file_name, pretty_fluent_file_serialized)
+            ru_fluent_file_path = self.create_ru_fluent_file(en_fluent_file_path)
+
+    def get_serialized_fluent_from_yaml_elements(self, yaml_elements):
+        fluent_serialized_messages = list(
+                map(lambda el: FluentSerializedMessage.from_yaml_element(el.id, el.name, FluentAstAttributeFactory.from_yaml_element(el), el.parent_id), yaml_elements)
+            )
+        fluent_exist_serialized_messages = list(filter(lambda m: m, fluent_serialized_messages))
+
+        if not len(fluent_exist_serialized_messages):
+            return None
+
+        return '\n'.join(fluent_exist_serialized_messages)
+
+    def create_en_fluent_file(self, relative_parent_dir, file_name, file_data):
+        en_new_dir_path = os.path.join(project.en_locale_prototypes_dir_path, relative_parent_dir)
+        en_fluent_file = FluentFile(os.path.join(en_new_dir_path, f'{file_name}.ftl'))
+        en_fluent_file.save_data(file_data)
+        logging.info(f'Актуализирован файл английской локали {en_fluent_file.full_path}')
+
+        return en_fluent_file.full_path
+
+    def create_ru_fluent_file(self, en_analog_file_path):
+        ru_file_full_path = en_analog_file_path.replace('en-US', 'ru-RU')
+
+        if os.path.isfile(ru_file_full_path):
+            return
+        else:
+            en_file = FluentFile(f'{en_analog_file_path}')
+            file = FluentFile(f'{ru_file_full_path}')
+            file.save_data(en_file.read_data())
+            logging.info(f'Создан файл русской локали {ru_file_full_path}')
+
+        return ru_file_full_path
+
+
+
+######################################## Var definitions ###############################################################
+
+logging.basicConfig(level = logging.INFO)
+project = Project()
+serializer = FluentSerializer()
+parser = FluentParser()
+formatter = FluentFormatter()
+
+yaml_files_paths = project.get_files_paths_by_dir(project.prototypes_dir_path, 'yml')
+yaml_files = list(map(lambda yaml_file_path: YAMLFile(yaml_file_path), yaml_files_paths))
+
+########################################################################################################################
+
+logging.info(f'Поиск yaml-файлов ...')
+YAMLExtractor(yaml_files).execute()

Tools/ss14_ru/yamlmodels.py

@@ -0,0 +1,26 @@
+class YAMLEntity:
+    def __init__(self, id, name, description, suffix, parent_id = None):
+        self.id = id
+        self.name = name
+        self.description = description
+        self.suffix = suffix
+        self.parent_id = parent_id
+
+
+class YAMLElements:
+    def __init__(self, items):
+        self.elements = list(map(lambda i: self.create_element(i), items))
+
+    def create_element(self, item):
+        if not 'id' in item:
+            return None
+
+        if item['type'] == 'entity':
+            entity = YAMLEntity(item['id'], item['name'] if 'name' in item else None,
+                                item['description'] if 'description' in item else None,
+                                item['suffix'] if 'suffix' in item else None,
+                                item['parent'] if 'parent' in item else None
+                                )
+            return entity
+        else:
+            return None