Python package with optional namespace sub-packages

Question:

Problem

I am struggling to create a single entry point for installing a python package that leverages namespace sub-package to allow users to optionally download additional modules. Below is the piece I am struggling with in this example. I have also provided additional context below as well to clarify the problem.

starwarssetup.py [Doesn’t work]

import setuptools

setuptools.setup(
    name="starwars",
    packages=setuptools.find_namespace_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    python_requires=">=3.7",
    install_requires=[
        'common'
    ],
    extra_requires={
        'characters': ['characters'],
        'weapons': ['weapons']
    }
)

$ pwd
> ~/starwars

$ pip install .
> ERROR: No matching distribution found for common

$ pip install .[characters]
> zsh: no matches found: .[characters]

$ pip install .[weapons]
> zsh: no matches found: .[weapons]

Project Goal

I am trying to create a python package with optional namespace sub-package dependencies that I can install from a private git repo. Below is an example of what the commands would look like.

# Installs only the common subpackage
$ pip install -e git+https://github.com/user/project.git#egg=starwars
# OR
$ $ pip install -e .

# Installs the common and characters subpackage
$ pip install -e git+https://github.com/user/project.git#egg=starwars[characters]
# OR
$ $ pip install -e .[characters]

# Installs only the common and weapons subpackage
$ pip install -e git+https://github.com/user/project.git#egg=starwars[weapons]
# OR
$ $ pip install -e .[weapons]

Project Structure

starwars
-- setup.py

-- common
  |-- setup.py
  |-- starwars
     |-- utils
     |-- abstract

-- characters (Optional)
  |-- setup.py
  |-- starwars
     |-- jedi
     |-- sith
     |-- senators

-- weapons (Optional)
  |-- setup.py
  |-- starwars
     |-- blaster
     |-- lightsabers

starwarscommonsetup.py

import setuptools


setuptools.setup(
    name="common",
    packages=setuptools.find_namespace_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    python_requires=">=3.7",
    install_requires=[
        "asyncio",
        "turtle"
    ]
)

starwarscharacterssetup.py

import setuptools


setuptools.setup(
    name="characters",
    packages=setuptools.find_namespace_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    python_requires=">=3.7",
    install_requires=['common']
)

starwarsweaponssetup.py

import setuptools


setuptools.setup(
    name="weapons",
    packages=setuptools.find_namespace_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    python_requires=">=3.7",
    install_requires=['common']
)

Current Status

I have successfully setup the native namespace sub-packages and can install them individually by using the commands below.

$ pwd
> ~/starwars

# Installing the common package
$ pushd ./common
$ pip install .
$ python -c 'import starwars.utils;'
$ popd

# Installing the characters package
$ pushd ./characters
$ pip install .
$ python -c 'import starwars.jedi;'
$ popd

# Installing the weapons package
$ pushd ./weapons
$ pip install .
$ python -c 'import starwars.lightsabers'
$ popd

References

Asked By: Will Udstrand

||
Source

Answers:

Here is the setup.py that worked for me. I got inspiration from this post

starwars/setup.py

import subprocess
from setuptools import setup
from setuptools.command.install import install


try:
    import pypandoc
    long_description = pypandoc.convert_file('README.md', 'rst')
except(IOError, ImportError):
    long_description = open('README.md').read()


class InstallLocalPackage(install):
    description = "Installs the subpackage specified"
    user_options = install.user_options + [
        ('extra=', None, '<extra to setup package with>'),
    ]

    def initialize_options(self):
        install.initialize_options(self)
        self.db = None

    def finalize_options(self):
        assert self.db in (None, 'characters', 'weapons'), 'Invalid extra!'
        install.finalize_options(self)

    @staticmethod
    def install_subpackage(subpackage_dir: str):
        dir_map = {
            'common': './common',
            'characters': './characters',
            'weapons': './weapons'
        }

        subprocess.call(
            f"pushd ./{dir_map[subpackage_dir]}; "
            f"pip install .;"
            f"popd;",
            shell=True
        )

    def install_subpackages(self):
        if self.db is None:
            [self.install_subpackage(package) for package in ['common', 'characters', 'weapons']]
        else:
            [self.install_subpackage(package) for package in ['common', self.db]]

    def run(self):
        install.run(self)
        self.install_subpackages()



setup(
    name="starwars",
    version_format='{tag}.{commits}',
    setup_requires=['very-good-setuptools-git-version'],
    author_email="[email protected]",
    description="Star Wars Python Package",
    long_description=long_description,
    long_description_content_type="text/markdown",
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    python_requires=">=3.7",
    cmdclass={ 'install': InstallLocalPackage },
    install_requires=[
        "pandas",
        "asyncio"
    ]
)
Answered By: Will Udstrand