[Buildroot] [PATCH v4 1/1] utils/scanpypi: add flit package support

[Yegor Yefremov]

On Mon, Nov 28, 2022 at 7:23 PM James Hilliard
<james.hilliard1 at gmail.com> wrote:
>
> These packages don't have a setup.py so we instead need to parse their
> pyproject.toml file.
>
> Note that this currently doesn't handle flit package dependency
> resolution.
>
> Signed-off-by: James Hilliard <james.hilliard1 at gmail.com>

Reviewed-by: Yegor Yefremov <yegorslists at googlemail.com>

> ---
> Changes v3 -> v4:
>   - prefer/support python3.11 tomllib
>   - use bool interpretation for None
> Changes v2 -> v3:
>   - minor cleanup
>   - rebase
>   - more detailed  commit log
> Changes v1 -> v2:
>   - remove homepage format fixes(sent as separate patch)
>   - remove load_setup fixes
> ---
>  utils/scanpypi | 85 ++++++++++++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 83 insertions(+), 2 deletions(-)
>
> diff --git a/utils/scanpypi b/utils/scanpypi
> index 3c98bb4bcc..38f0cb91e9 100755
> --- a/utils/scanpypi
> +++ b/utils/scanpypi
> @@ -42,6 +42,55 @@ except ImportError:
>            'pip install spdx_lookup')
>      liclookup = None
>
> +def toml_load(f):
> +    with open(f, 'rb') as fh:
> +        ex = None
> +
> +        # Try standard library tomllib first
> +        try:
> +            from tomllib import load
> +            return load(fh)
> +        except ImportError as e:
> +            pass
> +
> +        # Try regular tomli next
> +        try:
> +            from tomli import load
> +            return load(fh)
> +        except ImportError as e:
> +            ex = e
> +
> +        # Try pip's vendored tomli
> +        try:
> +            from pip._vendor.tomli import load
> +            try:
> +                return load(fh)
> +            except TypeError:
> +                # Fallback to handle older version
> +                try:
> +                    fh.seek(0)
> +                    w = io.TextIOWrapper(fh, encoding="utf8", newline="")
> +                    return load(w)
> +                finally:
> +                    w.detach()
> +        except ImportError as e:
> +            pass
> +
> +        # Try regular toml last
> +        try:
> +            from toml import load
> +            fh.seek(0)
> +            w = io.TextIOWrapper(fh, encoding="utf8", newline="")
> +            try:
> +                return load(w)
> +            finally:
> +                w.detach()
> +        except ImportError:
> +            pass
> +
> +        print('This package needs tomli')
> +        raise ex
> +
>
>  def setup_decorator(func, method):
>      """
> @@ -316,6 +365,35 @@ class BuildrootPackage():
>              os.chdir(current_dir)
>              sys.path.remove(self.tmp_extract)
>
> +    def load_pyproject(self):
> +        """
> +        Loads the corresponding pyproject.toml and store its metadata
> +        """
> +        current_dir = os.getcwd()
> +        os.chdir(self.tmp_extract)
> +        sys.path.insert(0, self.tmp_extract)
> +        try:
> +            pyproject_data = toml_load('pyproject.toml')
> +            try:
> +                self.setup_metadata = pyproject_data.get('project', {})
> +                self.metadata_name = self.setup_metadata.get('name', self.real_name)
> +                build_system = pyproject_data.get('build-system', {})
> +                build_backend = build_system.get('build-backend', None)
> +                if build_backend and build_backend == 'flit_core.buildapi':
> +                    self.setup_metadata['method'] = 'flit'
> +                elif build_system.get('backend-path', None):
> +                    self.setup_metadata['method'] = 'pep517'
> +                else:
> +                    self.setup_metadata['method'] = 'unknown'
> +            except KeyError:
> +                print('ERROR: Could not determine package metadata for {pkg}.\n'
> +                      .format(pkg=self.real_name))
> +                raise
> +        except FileNotFoundError:
> +            raise
> +        os.chdir(current_dir)
> +        sys.path.remove(self.tmp_extract)
> +
>      def get_requirements(self, pkg_folder):
>          """
>          Retrieve dependencies from the metadata found in the setup.py script of
> @@ -699,9 +777,12 @@ def main():
>              except ImportError as err:
>                  if 'buildutils' in str(err):
>                      print('This package needs buildutils')
> +                    continue
>                  else:
> -                    raise
> -                continue
> +                    try:
> +                        package.load_pyproject()
> +                    except Exception as e:
> +                        raise
>              except (AttributeError, KeyError) as error:
>                  print('Error: Could not install package {pkg}: {error}'.format(
>                      pkg=package.real_name, error=error))
> --
> 2.34.1
>