How to list all classes and methods/functions in a package – with a full folder/file path?
Question:
(for the purpose of better understanding the structure of package)
Let’s say we have some package/module (say, somemodule
), with the following structure:
somemodule
-- file1.py
fo(x) # a function
-- file2.py
bar(x) # a function
-- dir1
---- file3.py
A # a class
fun(y) # some method
_bo(y) # some "hidden" method
I would like some output like:
path name type
/file1.py fo(x) function
/file2.py bar(x) function
/dir1/file3.py A class
/dir1/file3.py A.fun(y) method
/dir1/file3.py A._bo(y) method
Given that I’m a novice in Python, I have no idea how to go about constructing such information, and would be happy for help.
The closest I found online was:
- How to list all functions in a module? – which didn’t help enough, since it only shows some functions (not the hidden ones), doesn’t show methods within classes, and doesn’t show file path (i.e.: neither dir, or help, or some of the other suggested functions seem to be up to the task).
- How do I list all files of a directory? – shows files, but not for a module (not the functions/classes/methods within the object)
Answers:
Heres an example on how to do it:
import inspect
import TestModule
def recusive_module_search(module):
members = inspect.getmembers(module)
for name, member in members:
if inspect.ismodule(member):
# Dont go too deep :)
if member is module:
recusive_module_search(member)
elif inspect.isfunction(member):
file = inspect.getfile(member)
print(file, function_signature_string(member), "function")
elif inspect.isclass(member):
file = inspect.getfile(member)
print(file, function_signature_string(member), "class")
class_members = inspect.getmembers(member)
for name, class_member in class_members:
if inspect.isfunction(class_member):
member_args = inspect.signature(class_member)
print(file, member.__name__ + "." + function_signature_string(class_member), "method")
def function_signature_string(member):
parameters = inspect.signature(member).parameters
return member.__name__ + "(" + ', '.join(str(x) for x in parameters.values()) + ")"
recusive_module_search(TestModule)
output:
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py Test(x: int, y: float) class
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py Test.__init__(self, x: int, y: float) method
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py Test.print_x(self) method
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py hi(x: int) function
C:UsersmcleasrcPydanticMongoEngineTestModuleSubModuletest.py test_fn(hello: str) function
Continue as desired 🙂
try dir(module) if there are sub modules you have to iterate and traverse in all classes
The above solution is great when the package references relevant functions/methods. But if they are not invoked by __init__
, then they won’t be listed.
After a bunch of digging I came up with the following solution. Probably not ideal, but worked for my use-case, so I’m re-sharing it here for others to use/improve upon.
def get_functions_and_methods(path):
"""
Given a .py file path - returns a list with all functions and methods in it.
Source: https://stackoverflow.com/q/73239026/256662
"""
import ast
with open(path) as file:
node = ast.parse(file.read())
def show_info(functionNode):
function_rep = ''
function_rep = functionNode.name + '('
for arg in functionNode.args.args:
function_rep += arg.arg + ','
function_rep = function_rep.rstrip(function_rep[-1])
function_rep += ')'
return function_rep
result = []
functions = [n for n in node.body if isinstance(n, ast.FunctionDef)]
classes = [n for n in node.body if isinstance(n, ast.ClassDef)]
for function in functions:
result.append(show_info(function))
for class_ in classes:
methods = [n for n in class_.body if isinstance(n, ast.FunctionDef)]
for method in methods:
result.append((class_.name + '.' + show_info(method)))
# print(', '.join(result))
return result
# This prints expected output
# fo(x), A.fun(self,y), A._bo(self,y), A.NS(y,z), B.foo(self,z), B._bar(self,t)
# Get paste to work (but it doesn't recycle :( )
# source: https://stackoverflow.com/a/35756195/256662
from functools import reduce
def _reduce_concat(x, sep=""):
return reduce(lambda x, y: str(x) + sep + str(y), x)
def paste(*lists, sep=" ", collapse=None):
result = map(lambda x: _reduce_concat(x, sep=sep), zip(*lists))
if collapse is not None:
return _reduce_concat(result, sep=collapse)
return list(result)
# this fails with recycling:
# paste("Hello", ["Ben", "Mike"]) # ['H Ben', 'e Mike'] # not what we want.
# paste(["Hello"], ["Ben", "Mike"]) # ['Hello Ben'] # not what we want.
# paste("a", ["Ben", "Mike"]) # ['a Ben'] # not what we want.
# gets all the py files from a root folder (excluding tests)
def get_all_py_files_no_tests(root):
result = []
# based on: https://stackoverflow.com/a/2909998/256662
for path, subdirs, files in os.walk(root):
for name in files:
if name[-3:] == '.py' and ('tests' not in path):
result.append(os.path.join(path, name))
# print(os.path.join(path, name))
return result
def merge_py_file_and_funcs(py_file):
import numpy as np
funcs = get_functions_and_methods(py_file)
py_file_recycled = np.repeat(py_file, len(funcs))
return paste(py_file_recycled, funcs)
def flatten(l):
"""
# source: https://stackoverflow.com/a/952952/256662
"""
return [item for sublist in l for item in sublist]
def get_all_fun_from_root(root):
py_files = get_all_py_files_no_tests(root)
all_combos = flatten([merge_py_file_and_funcs(py_file) for py_file in py_files])
return all_combos
# source: https://stackoverflow.com/a/3136703/256662
def search_replace_in_list(words, search = "", replace = ""):
return [w.replace(search, replace) for w in words]
# search_replace_in_list(["abs", "abbfe"], "b", "_b_")
# ['a_b_s', 'a_b__b_fe']
def given_pkg_return_funs_and_methods(pkg):
if type(pkg) is str:
pkg_folder = pkg
else:
# Source: https://stackoverflow.com/a/12154601/256662
import os
import inspect
pkg_folder = os.path.dirname(inspect.getfile(pkg))
all_items = get_all_fun_from_root(pkg_folder)
cleaned_all_items = search_replace_in_list(all_items, pkg_folder)
return cleaned_all_items
Example of usage:
import numpy
given_pkg_return_funs_and_methods(numpy)
This returns:
['/__config__.py get_info(name)',
'/__config__.py show)',
'/_globals.py _NoValueType.__new__(cls)',
'/_globals.py _NoValueType.__reduce__(self)',
'/_globals.py _NoValueType.__repr__(self)',
'/_pytesttester.py _show_numpy_info)',
'/_pytesttester.py PytestTester.__init__(self,module_name)',
'/_pytesttester.py PytestTester.__call__(self,label,verbose,extra_argv,doctests,coverage,durations,tests)',
'/conftest.py pytest_configure(config)',
'/conftest.py pytest_addoption(parser)',
'/conftest.py pytest_sessionstart(session)',
'/conftest.py pytest_itemcollected(item)',
'/conftest.py check_fpu_mode(request)',
'/conftest.py add_np(doctest_namespace)',
'/ctypeslib.py _num_fromflags(flaglist)',
'/ctypeslib.py _flags_fromnum(num)',
##### Etc...
Nicer printing can be done using something like:
import some_package # or use `some_package = adirectpath`
all_fun = given_pkg_return_funs_and_methods(some_package)
for i in all_fun:
print(i)
This solution used the following stackoverflow references:
(for the purpose of better understanding the structure of package)
Let’s say we have some package/module (say, somemodule
), with the following structure:
somemodule
-- file1.py
fo(x) # a function
-- file2.py
bar(x) # a function
-- dir1
---- file3.py
A # a class
fun(y) # some method
_bo(y) # some "hidden" method
I would like some output like:
path name type
/file1.py fo(x) function
/file2.py bar(x) function
/dir1/file3.py A class
/dir1/file3.py A.fun(y) method
/dir1/file3.py A._bo(y) method
Given that I’m a novice in Python, I have no idea how to go about constructing such information, and would be happy for help.
The closest I found online was:
- How to list all functions in a module? – which didn’t help enough, since it only shows some functions (not the hidden ones), doesn’t show methods within classes, and doesn’t show file path (i.e.: neither dir, or help, or some of the other suggested functions seem to be up to the task).
- How do I list all files of a directory? – shows files, but not for a module (not the functions/classes/methods within the object)
Heres an example on how to do it:
import inspect
import TestModule
def recusive_module_search(module):
members = inspect.getmembers(module)
for name, member in members:
if inspect.ismodule(member):
# Dont go too deep :)
if member is module:
recusive_module_search(member)
elif inspect.isfunction(member):
file = inspect.getfile(member)
print(file, function_signature_string(member), "function")
elif inspect.isclass(member):
file = inspect.getfile(member)
print(file, function_signature_string(member), "class")
class_members = inspect.getmembers(member)
for name, class_member in class_members:
if inspect.isfunction(class_member):
member_args = inspect.signature(class_member)
print(file, member.__name__ + "." + function_signature_string(class_member), "method")
def function_signature_string(member):
parameters = inspect.signature(member).parameters
return member.__name__ + "(" + ', '.join(str(x) for x in parameters.values()) + ")"
recusive_module_search(TestModule)
output:
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py Test(x: int, y: float) class
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py Test.__init__(self, x: int, y: float) method
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py Test.print_x(self) method
C:UsersmcleasrcPydanticMongoEngineTestModulefunctions.py hi(x: int) function
C:UsersmcleasrcPydanticMongoEngineTestModuleSubModuletest.py test_fn(hello: str) function
Continue as desired 🙂
try dir(module) if there are sub modules you have to iterate and traverse in all classes
The above solution is great when the package references relevant functions/methods. But if they are not invoked by __init__
, then they won’t be listed.
After a bunch of digging I came up with the following solution. Probably not ideal, but worked for my use-case, so I’m re-sharing it here for others to use/improve upon.
def get_functions_and_methods(path):
"""
Given a .py file path - returns a list with all functions and methods in it.
Source: https://stackoverflow.com/q/73239026/256662
"""
import ast
with open(path) as file:
node = ast.parse(file.read())
def show_info(functionNode):
function_rep = ''
function_rep = functionNode.name + '('
for arg in functionNode.args.args:
function_rep += arg.arg + ','
function_rep = function_rep.rstrip(function_rep[-1])
function_rep += ')'
return function_rep
result = []
functions = [n for n in node.body if isinstance(n, ast.FunctionDef)]
classes = [n for n in node.body if isinstance(n, ast.ClassDef)]
for function in functions:
result.append(show_info(function))
for class_ in classes:
methods = [n for n in class_.body if isinstance(n, ast.FunctionDef)]
for method in methods:
result.append((class_.name + '.' + show_info(method)))
# print(', '.join(result))
return result
# This prints expected output
# fo(x), A.fun(self,y), A._bo(self,y), A.NS(y,z), B.foo(self,z), B._bar(self,t)
# Get paste to work (but it doesn't recycle :( )
# source: https://stackoverflow.com/a/35756195/256662
from functools import reduce
def _reduce_concat(x, sep=""):
return reduce(lambda x, y: str(x) + sep + str(y), x)
def paste(*lists, sep=" ", collapse=None):
result = map(lambda x: _reduce_concat(x, sep=sep), zip(*lists))
if collapse is not None:
return _reduce_concat(result, sep=collapse)
return list(result)
# this fails with recycling:
# paste("Hello", ["Ben", "Mike"]) # ['H Ben', 'e Mike'] # not what we want.
# paste(["Hello"], ["Ben", "Mike"]) # ['Hello Ben'] # not what we want.
# paste("a", ["Ben", "Mike"]) # ['a Ben'] # not what we want.
# gets all the py files from a root folder (excluding tests)
def get_all_py_files_no_tests(root):
result = []
# based on: https://stackoverflow.com/a/2909998/256662
for path, subdirs, files in os.walk(root):
for name in files:
if name[-3:] == '.py' and ('tests' not in path):
result.append(os.path.join(path, name))
# print(os.path.join(path, name))
return result
def merge_py_file_and_funcs(py_file):
import numpy as np
funcs = get_functions_and_methods(py_file)
py_file_recycled = np.repeat(py_file, len(funcs))
return paste(py_file_recycled, funcs)
def flatten(l):
"""
# source: https://stackoverflow.com/a/952952/256662
"""
return [item for sublist in l for item in sublist]
def get_all_fun_from_root(root):
py_files = get_all_py_files_no_tests(root)
all_combos = flatten([merge_py_file_and_funcs(py_file) for py_file in py_files])
return all_combos
# source: https://stackoverflow.com/a/3136703/256662
def search_replace_in_list(words, search = "", replace = ""):
return [w.replace(search, replace) for w in words]
# search_replace_in_list(["abs", "abbfe"], "b", "_b_")
# ['a_b_s', 'a_b__b_fe']
def given_pkg_return_funs_and_methods(pkg):
if type(pkg) is str:
pkg_folder = pkg
else:
# Source: https://stackoverflow.com/a/12154601/256662
import os
import inspect
pkg_folder = os.path.dirname(inspect.getfile(pkg))
all_items = get_all_fun_from_root(pkg_folder)
cleaned_all_items = search_replace_in_list(all_items, pkg_folder)
return cleaned_all_items
Example of usage:
import numpy
given_pkg_return_funs_and_methods(numpy)
This returns:
['/__config__.py get_info(name)',
'/__config__.py show)',
'/_globals.py _NoValueType.__new__(cls)',
'/_globals.py _NoValueType.__reduce__(self)',
'/_globals.py _NoValueType.__repr__(self)',
'/_pytesttester.py _show_numpy_info)',
'/_pytesttester.py PytestTester.__init__(self,module_name)',
'/_pytesttester.py PytestTester.__call__(self,label,verbose,extra_argv,doctests,coverage,durations,tests)',
'/conftest.py pytest_configure(config)',
'/conftest.py pytest_addoption(parser)',
'/conftest.py pytest_sessionstart(session)',
'/conftest.py pytest_itemcollected(item)',
'/conftest.py check_fpu_mode(request)',
'/conftest.py add_np(doctest_namespace)',
'/ctypeslib.py _num_fromflags(flaglist)',
'/ctypeslib.py _flags_fromnum(num)',
##### Etc...
Nicer printing can be done using something like:
import some_package # or use `some_package = adirectpath`
all_fun = given_pkg_return_funs_and_methods(some_package)
for i in all_fun:
print(i)
This solution used the following stackoverflow references: