anaconda/anaconda-40.22.3.13/pyanaconda/localization.py

# Localization classes and functions
#
# Copyright (C) 2012-2013 Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#

import gettext
import os
import re
import langtable
import locale as locale_mod
import glob
from collections import namedtuple
import functools

from pyanaconda.core import constants
from pyanaconda.core.util import setenv, execWithRedirect
from pyanaconda.core.string import upcase_first_letter
from pyanaconda.modules.common.constants.services import BOSS

from pyanaconda.anaconda_loggers import get_module_logger
log = get_module_logger(__name__)

SCRIPTS_SUPPORTED_BY_CONSOLE = {'Latn', 'Cyrl', 'Grek'}


class LocalizationConfigError(Exception):
    """Exception class for localization configuration related problems"""

    pass


class InvalidLocaleSpec(LocalizationConfigError):
    """Exception class for the errors related to invalid locale specs"""

    pass


def is_valid_langcode(langcode):
    """Check if the given locale has a language specified.

    :return: whether the language or locale is valid
    :rtype: bool
    """
    parsed = langtable.parse_locale(langcode)
    return bool(parsed.language)


def raise_on_invalid_locale(arg):
    """Helper to abort when a locale is not valid.

    :raise: InvalidLocaleSpec
    """
    if not is_valid_langcode(arg):
        raise InvalidLocaleSpec("'{}' is not a valid locale".format(arg))


def get_language_id(locale):
    """Return language id without territory or anything else."""
    return langtable.parse_locale(locale).language


@functools.cache
def get_common_languages():
    """Return common languages to prioritize them"""
    return langtable.list_common_languages()


def is_supported_locale(locale):
    """Function that tells if the given locale is supported by the Anaconda or
    not. We consider locales supported by the langtable as supported by the
    Anaconda.

    :param locale: locale to test
    :type locale: str
    :return: whether the given locale is supported or not
    :rtype: bool
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    en_name = get_english_name(locale)
    return bool(en_name)


def locale_supported_in_console(locale):
    """Function that tells if the given locale can be displayed by the Linux console.

    The Linux console can display Latin, Cyrillic and Greek characters reliably,
    but others such as Japanese, can't be correctly installed.

    :param str locale: locale to test
    :return: whether the given locale is supported by the console or not
    :rtype: bool
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    locale_scripts = get_locale_scripts(locale)
    return set(locale_scripts).issubset(SCRIPTS_SUPPORTED_BY_CONSOLE)


def find_best_locale_match(locale, langcodes):
    """Find the best match for the locale in a list of langcodes. This is useful
    when e.g. pt_BR is a locale and there are possibilities to choose an item
    (e.g. rnote) for a list containing both pt and pt_BR or even also pt_PT.

    :param locale: a valid locale (e.g. en_US.UTF-8 or sr_RS.UTF-8@latin, etc.)
    :type locale: str
    :param langcodes: a list or generator of langcodes (e.g. en, en_US, en_US@latin, etc.)
    :type langcodes: list(str) or generator(str)
    :return: the best matching langcode from the list of None if none matches
    :rtype: str or None
    """
    # Parse the locale.
    if not is_valid_langcode(locale):
        return None

    locale_parsed = langtable.parse_locale(locale)

    # Get a score for each langcode.
    scores = {}

    for langcode in langcodes:
        if not is_valid_langcode(langcode):
            continue

        langcode_parsed = langtable.parse_locale(langcode)

        # Don't match a non-POSIX locale with a POSIX langcode.
        if langcode_parsed.variant == "POSIX" and locale_parsed.variant != "POSIX":
            continue

        score = _evaluate_locales(locale_parsed, langcode_parsed)

        # Matches matching only script or encoding or both are not useful.
        # The score of 100 requires at least territory to have matched.
        if score <= 100:
            continue

        scores[langcode] = score

    # Find the best one.
    return max(scores.keys(), key=scores.get, default=None)


def _evaluate_locales(locale, langcode):
    """Compare a locale with a langcode.

    :param locale: a parsed locale
    :param langcode: a parsed langcode
    :return: a score
    """
    return \
        _evaluate_values(locale.language, langcode.language, 1000) + \
        _evaluate_values(locale.territory, langcode.territory, 100) + \
        _evaluate_values(locale.script, langcode.script, 10) + \
        _evaluate_values(locale.variant, langcode.variant, 10) + \
        _evaluate_values(locale.encoding, langcode.encoding, 1)


def _evaluate_values(locale_value, langcode_value, weight):
    """Compare a locale value with a langcode value.

    :param locale_value: an item of the parsed locale
    :param langcode_value: an item of the parsed langcode
    :return: a score
    """
    if locale_value and langcode_value:
        if locale_value == langcode_value:
            # Match.
            return weight
        else:
            # Not match.
            return -weight

    if langcode_value and not locale_value:
        # The langcode has something the locale doesn't have.
        return -weight

    return 0


def setup_locale(locale, localization_proxy=None, text_mode=False):
    """Procedure setting the system to use the given locale and store it in to the
    localization module (if given). DOES NOT PERFORM ANY CHECKS OF THE GIVEN
    LOCALE.

    $LANG must be set by the caller in order to set the language used by gettext.
    Doing this in a thread-safe way is up to the caller.

    We also try to set a proper console font for the locale in text mode.
    If the font for the locale can't be displayed in the Linux console,
    we fall back to the English locale.

    This function returns the locale that was used in the setlocale call, which,
    depending on what the environment and interface is able to support, may be
    different from the locale requested.

    :param str locale: locale to setup
    :param localization_proxy: DBus proxy of the localization module or None
    :param bool text_mode: if the locale is being setup for text mode
    :return: the locale that was actually set
    :rtype: str
    """
    if localization_proxy:
        localization_proxy.Language = locale

    # not all locales might be displayable in text mode
    if text_mode:
        # check if the script corresponding to the locale/language
        # can be displayed by the Linux console
        # * all scripts for the given locale/language need to be
        #   supported by the linux console
        # * otherwise users might get a screen full of white rectangles
        #   (also known as "tofu") in text mode
        # then we also need to check if we have information about what
        # font to use for correctly displaying the given language/locale

        script_supported = locale_supported_in_console(locale)
        log.debug("scripts found for locale %s: %s", locale, get_locale_scripts(locale))

        console_fonts = get_locale_console_fonts(locale)
        log.debug("console fonts found for locale %s: %s", locale, console_fonts)

        font_set = False
        if script_supported and console_fonts:
            # try to set console font
            for font in console_fonts:
                if set_console_font(font):
                    # console font set successfully, skip the rest
                    font_set = True
                    break

        if not font_set:
            log.warning("can't set console font for locale %s", locale)
            # report what exactly went wrong
            if not script_supported:
                log.warning("script not supported by console for locale %s", locale)
            if not console_fonts:  # no fonts known for locale
                log.warning("no console font found for locale %s", locale)
            if script_supported and console_fonts:
                log.warning("none of the suggested fonts can be set for locale %s", locale)
            log.warning("falling back to the English locale")
            locale = constants.DEFAULT_LANG
            os.environ["LANG"] = locale  # pylint: disable=environment-modify

    # set the locale to the value we have selected
    # Since glibc does not install all locales, an installable locale may not
    # actually be available right now. Give it a shot and fallback.
    log.debug("setting locale to: %s", locale)
    setenv("LANG", locale)

    try:
        locale_mod.setlocale(locale_mod.LC_ALL, locale)
    except locale_mod.Error as e:
        log.debug("setlocale failed: %s", e)
        locale = constants.DEFAULT_LANG
        setenv("LANG", locale)
        locale_mod.setlocale(locale_mod.LC_ALL, locale)

    set_modules_locale(locale)

    return locale


def set_modules_locale(locale):
    """Set locale of all modules."""
    boss_proxy = BOSS.get_proxy()
    boss_proxy.SetLocale(locale)


def get_english_name(locale):
    """Function returning english name for the given locale.

    :param locale: locale to return english name for
    :type locale: str
    :return: english name for the locale or empty string if unknown
    :rtype: st
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    name = langtable.language_name(languageId=locale, languageIdQuery="en")
    return upcase_first_letter(name)


def get_native_name(locale):
    """Function returning native name for the given locale.

    :param locale: locale to return native name for
    :type locale: str
    :return: english name for the locale or empty string if unknown
    :rtype: st
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.language_name(languageId=locale)


def get_available_translations(localedir=None):
    """Method that generates (i.e. returns a generator) available translations
    for the installer in the given localedir.

    :type localedir: str
    :return: generator yielding available translations (languages)
    :rtype: generator yielding strings
    """
    localedir = localedir or gettext._default_localedir

    # usually there are no message files for en
    messagefiles = sorted(glob.glob(localedir + "/*/LC_MESSAGES/anaconda.mo") +
                          ["blob/en/blob/blob"])
    trans_gen = (path.split(os.path.sep)[-3] for path in messagefiles)

    langs = set()

    for trans in trans_gen:
        lang = get_language_id(trans)
        if lang and lang not in langs:
            langs.add(lang)
            # check if there are any locales for the language
            locales = get_language_locales(lang)
            if not locales:
                continue

            yield lang


@functools.lru_cache(2048)
def locale_has_translation(locale):
    """Does the locale have a translation available?

    Checks if a given locale will receive a gettext translation. That could be either because the
    locale has a translation, or because there is another translation to fall back onto. In
    reality, the fallback is mostly of the type "ja_JP" -> "ja".

    For English, always return true, because that is the "untranslated" state which does not need
    translation files present to work.

    :param str locale: locale to check
    :return bool: is there a translation
    """
    if get_language_id(locale) == "en":
        return True

    files = gettext.find("anaconda", None, [locale], True)
    return bool(files)


def get_language_locales(lang):
    """Function returning all locales available for the given language.

    :param lang: language to get available locales for
    :type lang: str
    :return: a list of available locales
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(lang)

    return langtable.list_locales(languageId=lang)


def get_territory_locales(territory):
    """Function returning list of locales for the given territory. The list is
    sorted from the most probable locale to the least probable one (based on
    langtable's ranking.

    :param territory: territory to return locales for
    :type territory: str
    :return: list of locales
    :rtype: list of strings
    """
    return langtable.list_locales(territoryId=territory)


def get_locale_keyboards(locale):
    """Function returning preferred keyboard layouts for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred keyboard layouts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_keyboards(languageId=locale)


def get_common_keyboard_layouts():
    """Function returning common keyboard layouts carrying high ranks.

    :return: list of common keyboard layouts
    :rtype: list of strings
    """
    return langtable.list_common_keyboards()


def get_locale_timezones(locale):
    """Function returning preferred timezones for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred timezones
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_timezones(languageId=locale)


def get_locale_console_fonts(locale):
    """Function returning preferred console fonts for the given locale.

    :param str locale: locale string
    :return: list of preferred console fonts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_consolefonts(languageId=locale)


def get_locale_scripts(locale):
    """Function returning preferred scripts (writing systems) for the given locale.

    :param locale: locale string
    :type locale: str
    :return: list of preferred scripts
    :rtype: list of strings
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    raise_on_invalid_locale(locale)

    return langtable.list_scripts(languageId=locale)


def get_xlated_timezone(tz_spec_part):
    """Function returning translated name of a region, city or complete timezone
    name according to the current value of the $LANG variable.

    :param tz_spec_part: a region, city or complete timezone name
    :type tz_spec_part: str
    :return: translated name of the given region, city or timezone
    :rtype: str
    :raise InvalidLocaleSpec: if an invalid locale is given (see is_valid_langcode)
    """
    locale = os.environ.get("LANG", constants.DEFAULT_LANG)

    raise_on_invalid_locale(locale)

    xlated = langtable.timezone_name(tz_spec_part, languageIdQuery=locale)
    return xlated


def get_firmware_language(text_mode=False):
    """Procedure that returns the firmware language information (if any).

    :param boot text_mode: if the locale is being setup for text mode
    :return: the firmware language translated into a locale string, or None
    :rtype: str
    """
    try:
        n = "/sys/firmware/efi/efivars/PlatformLang-8be4df61-93ca-11d2-aa0d-00e098032b8c"
        with open(n, 'r') as f:
            d = f.read()
    except OSError:
        return None

    # the contents of the file are:
    # 4-bytes of attribute data that we don't care about
    # NUL terminated ASCII string like 'en-US'.
    if len(d) < 10:
        log.debug("PlatformLang was too short")
        return None
    d = d[4:]
    if d[2] != '-':
        log.debug("PlatformLang was malformed")
        return None

    # they use - and we use _, so fix it...
    d = d[:2] + '_' + d[3:-1]

    # UEFI 2.3.1 Errata C specifies 2 aliases in common use that
    # aren't part of RFC 4646, but are allowed in PlatformLang.
    # Because why make anything simple?
    if d.startswith('zh_chs'):
        d = 'zh_Hans'
    elif d.startswith('zh_cht'):
        d = 'zh_Hant'
    d += '.UTF-8'

    if not is_supported_locale(d):
        log.debug("PlatformLang was '%s', which is unsupported.", d)
        return None

    locales = get_language_locales(d)
    if not locales:
        log.debug("No locales found for the PlatformLang '%s'.", d)
        return None

    log.debug("Using UEFI PlatformLang '%s' ('%s') as our language.", d, locales[0])
    return locales[0]


_DateFieldSpec = namedtuple("DateFieldSpec", ["format", "suffix"])


def resolve_date_format(year, month, day, fail_safe=True):
    """Puts the year, month and day objects in the right order according to the
    currently set locale and provides format specification for each of the
    fields.

    :param year: any object or value representing year
    :type year: any
    :param month: any object or value representing month
    :type month: any
    :param day: any object or value representing day
    :type day: any
    :param bool fail_safe: whether to fall back to default in case of invalid
                           format or raise exception instead
    :returns: a pair where the first field contains a tuple with the year, month
              and day objects/values put in the right order and where the second
              field contains a tuple with three :class:`_DateFieldSpec` objects
              specifying formats respectively to the first (year, month, day)
              field, e.g. ((year, month, day), (y_fmt, m_fmt, d_fmt))
    :rtype: tuple
    :raise ValueError: in case currently set locale has unsupported date
                       format and fail_safe is set to False
    """
    FAIL_SAFE_DEFAULT = "%Y-%m-%d"

    def order_terms_formats(fmt_str):
        # see date (1), 'O' (not '0') is a mystery, 'E' is Buddhist calendar, '(.*)'
        # is an arbitrary suffix
        field_spec_re = re.compile(r'([-_0OE^#]*)([yYmbBde])(.*)')

        # see date (1)
        fmt_str = fmt_str.replace("%F", "%Y-%m-%d")

        # e.g. "%d.%m.%Y" -> ['d.', 'm.', 'Y']
        fields = fmt_str.split("%")[1:]

        ordered_terms = []
        ordered_formats = []
        for field in fields:
            match = field_spec_re.match(field)
            if not match:
                # ignore fields we are not interested in (like %A for weekday name, etc.)
                continue

            prefix, item, suffix = match.groups()
            if item in ("d", "e"):
                # "e" is the same as "_d"
                ordered_terms.append(day)
            elif item in ("Y", "y"):
                # 4-digit year, 2-digit year
                ordered_terms.append(year)
            elif item in ("m", "b", "B"):
                # month number, short month name, long month name
                ordered_terms.append(month)

            # "%" + prefix + item gives a format for date/time formatting functions
            ordered_formats.append(_DateFieldSpec("%" + prefix + item, suffix.strip()))

        if len(ordered_terms) != 3 or len(ordered_formats) != 3:
            raise ValueError("Not all fields successfully identified in the format '%s'" % fmt_str)

        return (tuple(ordered_terms), tuple(ordered_formats))

    fmt_str = locale_mod.nl_langinfo(locale_mod.D_FMT)

    if not fmt_str or "%" not in fmt_str:
        if fail_safe:
            # use some sane default
            fmt_str = FAIL_SAFE_DEFAULT
        else:
            raise ValueError("Invalid date format string for current locale: '%s'" % fmt_str)

    try:
        return order_terms_formats(fmt_str)
    except ValueError:
        if not fail_safe:
            raise
        else:
            # if this call fails too, something is going terribly wrong and we
            # should be informed about it
            return order_terms_formats(FAIL_SAFE_DEFAULT)


def set_console_font(font):
    """Try to set console font to the given value.

    :param str font: console font name
    :returns: True on success, False on failure
    :rtype: bool
    """
    log.debug("setting console font to %s", font)
    rc = execWithRedirect("setfont", [font])
    if rc == 0:
        log.debug("console font set successfully to %s", font)
        return True
    else:
        log.error("setting console font to %s failed", font)
        return False


def setup_locale_environment(locale=None, text_mode=False, prefer_environment=False):
    """Clean and configure the local environment variables.

    This function will attempt to determine the desired locale and configure
    the process environment (os.environ) in the least surprising way. If a
    locale argument is provided, it will be attempted first. After that, this
    function will attempt to use the language environment variables in a manner
    similar to gettext(3) (in order, $LANGUAGE, $LC_ALL, $LC_MESSAGES, $LANG),
    followed by the UEFI PlatformLang, followed by a default.

    When this function returns, $LANG will be set, and $LANGUAGE, $LC_ALL,
    and $LC_MESSAGES will not be set, because they get in the way when changing
    the language after startup.

    This function must be run before any threads are started. This function
    modifies the process environment, which is not thread-safe.

    :param str locale: locale to setup if provided
    :param bool text_mode: if the locale is being setup for text mode
    :param bool prefer_environment: whether the process environment, if available, overrides the locale parameter
    :return: None
    :rtype: None
    """
    # pylint: disable=environment-modify

    # Look for a locale in the environment. If the variable is setup but
    # empty it doesn't count, and some programs (KDE) actually do this.
    # If prefer_environment is set, the environment locale can override
    # the parameter passed in. This can be used, for example, by initial-setup,
    # to prefer the possibly-more-recent environment settings before falling back
    # to a locale set at install time and saved in the kickstart.
    if not locale or prefer_environment:
        for varname in ("LANGUAGE", "LC_ALL", "LC_MESSAGES", "LANG"):
            if varname in os.environ and os.environ[varname]:
                locale = os.environ[varname]
                break

    # Look for a locale in the firmware if there was nothing in the environment
    if not locale:
        locale = get_firmware_language(text_mode)

    # parse the locale using langtable
    if locale:
        try:
            env_langs = get_language_locales(locale)
            # the first language is the best match
            locale = env_langs[0]
        except (InvalidLocaleSpec, IndexError):
            log.error("Invalid locale '%s' given on command line, kickstart or environment", locale)
            locale = None

    # If langtable returned no locales, or if nothing was configured, fall back to the default
    if not locale:
        locale = constants.DEFAULT_LANG

    # Save the locale in the environment
    os.environ["LANG"] = locale

    # Cleanup the rest of the environment variables
    for varname in ("LANGUAGE", "LC_ALL", "LC_MESSAGES"):
        if varname in os.environ:
            del os.environ[varname]