How to copy directory recursively in python and overwrite all?
Question:
I’m trying to copy /home/myUser/dir1/ and all its contents (and their contents, etc.) to /home/myuser/dir2/ in python. Furthermore, I want the copy to overwrite everything in dir2/.
It looks like distutils.dir_util.copy_tree might be the right tool for the job, but not sure if there’s anything easier/more obvious to use for such a simple task.
If it is the right tool, how do I use it? According to the docs there are 8 parameters that it takes. Do I have to pass all 8 are just src, dst and update, and if so, how (I’m brand new to Python).
If there’s something out there that’s better, can someone give me an example and point me in the right direction?
Answers:
Notice:
distutils has been deprecated and will be removed in Python 3.12. Consider looking for other answers at this question if you are looking for a post-3.12 solution.
Original answer:
You can use distutils.dir_util.copy_tree. It works just fine and you don’t have to pass every argument, only src and dst are mandatory.
However in your case you can’t use a similar tool likeshutil.copytree because it behaves differently: as the destination directory must not exist this function can’t be used for overwriting its contents.
If you want to use the cp tool as suggested in the question comments beware that using the subprocess module is currently the recommended way for spawning new processes as you can see in the documentation of the os.system function.
Have a look at the shutil package, especially rmtree and copytree. You can check if a file / path exists with os.paths.exists(<path>).
import shutil
import os
def copy_and_overwrite(from_path, to_path):
if os.path.exists(to_path):
shutil.rmtree(to_path)
shutil.copytree(from_path, to_path)
Vincent was right about copytree not working, if dirs already exist. So distutils is the nicer version. Below is a fixed version of shutil.copytree. It’s basically copied 1-1, except the first os.makedirs() put behind an if-else-construct:
import os
from shutil import *
def copytree(src, dst, symlinks=False, ignore=None):
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
if not os.path.isdir(dst): # This one line does the trick
os.makedirs(dst)
errors = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
copytree(srcname, dstname, symlinks, ignore)
else:
# Will raise a SpecialFileError for unsupported file types
copy2(srcname, dstname)
# catch the Error from the recursive copytree so that we can
# continue with other files
except Error, err:
errors.extend(err.args[0])
except EnvironmentError, why:
errors.append((srcname, dstname, str(why)))
try:
copystat(src, dst)
except OSError, why:
if WindowsError is not None and isinstance(why, WindowsError):
# Copying file access times may fail on Windows
pass
else:
errors.extend((src, dst, str(why)))
if errors:
raise Error, errors
Here’s a simple solution to recursively overwrite a destination with a source, creating any necessary directories as it goes. This does not handle symlinks, but it would be a simple extension (see answer by @Michael above).
def recursive_overwrite(src, dest, ignore=None):
if os.path.isdir(src):
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(src)
if ignore is not None:
ignored = ignore(src, files)
else:
ignored = set()
for f in files:
if f not in ignored:
recursive_overwrite(os.path.join(src, f),
os.path.join(dest, f),
ignore)
else:
shutil.copyfile(src, dest)
In Python 3.8 the dirs_exist_ok keyword argument was added to shutil.copytree():
dirs_exist_ok dictates whether to raise an exception in case dst or any missing parent directory already exists.
So, the following will work in recent versions of Python, even if the destination directory already exists:
shutil.copytree(src, dest, dirs_exist_ok=True) # 3.8+ only!
One major benefit is that it’s more flexible than distutils.dir_util.copy_tree(), as it takes additional arguments on files to ignore, etc. (see the documentation). On top of that, the accepted PEP 632 also states that distutils will be deprecated and subsequently removed in future versions of Python 3.
My simple answer.
def get_files_tree(src="src_path"):
req_files = []
for r, d, files in os.walk(src):
for file in files:
src_file = os.path.join(r, file)
src_file = src_file.replace('\', '/')
if src_file.endswith('.db'):
continue
req_files.append(src_file)
return req_files
def copy_tree_force(src_path="",dest_path=""):
"""
make sure that all the paths has correct slash characters.
"""
for cf in get_files_tree(src=src_path):
df= cf.replace(src_path, dest_path)
if not os.path.exists(os.path.dirname(df)):
os.makedirs(os.path.dirname(df))
shutil.copy2(cf, df)
There has been an update to shutil.copytree(). You can add the following argument:
shutil.copytree(src, dst, dirs_exist_ok=True)
I’m trying to copy /home/myUser/dir1/ and all its contents (and their contents, etc.) to /home/myuser/dir2/ in python. Furthermore, I want the copy to overwrite everything in dir2/.
It looks like distutils.dir_util.copy_tree might be the right tool for the job, but not sure if there’s anything easier/more obvious to use for such a simple task.
If it is the right tool, how do I use it? According to the docs there are 8 parameters that it takes. Do I have to pass all 8 are just src, dst and update, and if so, how (I’m brand new to Python).
If there’s something out there that’s better, can someone give me an example and point me in the right direction?
Notice:
distutils has been deprecated and will be removed in Python 3.12. Consider looking for other answers at this question if you are looking for a post-3.12 solution.
Original answer:
You can use distutils.dir_util.copy_tree. It works just fine and you don’t have to pass every argument, only src and dst are mandatory.
However in your case you can’t use a similar tool likeshutil.copytree because it behaves differently: as the destination directory must not exist this function can’t be used for overwriting its contents.
If you want to use the cp tool as suggested in the question comments beware that using the subprocess module is currently the recommended way for spawning new processes as you can see in the documentation of the os.system function.
Have a look at the shutil package, especially rmtree and copytree. You can check if a file / path exists with os.paths.exists(<path>).
import shutil
import os
def copy_and_overwrite(from_path, to_path):
if os.path.exists(to_path):
shutil.rmtree(to_path)
shutil.copytree(from_path, to_path)
Vincent was right about copytree not working, if dirs already exist. So distutils is the nicer version. Below is a fixed version of shutil.copytree. It’s basically copied 1-1, except the first os.makedirs() put behind an if-else-construct:
import os
from shutil import *
def copytree(src, dst, symlinks=False, ignore=None):
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
if not os.path.isdir(dst): # This one line does the trick
os.makedirs(dst)
errors = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
copytree(srcname, dstname, symlinks, ignore)
else:
# Will raise a SpecialFileError for unsupported file types
copy2(srcname, dstname)
# catch the Error from the recursive copytree so that we can
# continue with other files
except Error, err:
errors.extend(err.args[0])
except EnvironmentError, why:
errors.append((srcname, dstname, str(why)))
try:
copystat(src, dst)
except OSError, why:
if WindowsError is not None and isinstance(why, WindowsError):
# Copying file access times may fail on Windows
pass
else:
errors.extend((src, dst, str(why)))
if errors:
raise Error, errors
Here’s a simple solution to recursively overwrite a destination with a source, creating any necessary directories as it goes. This does not handle symlinks, but it would be a simple extension (see answer by @Michael above).
def recursive_overwrite(src, dest, ignore=None):
if os.path.isdir(src):
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(src)
if ignore is not None:
ignored = ignore(src, files)
else:
ignored = set()
for f in files:
if f not in ignored:
recursive_overwrite(os.path.join(src, f),
os.path.join(dest, f),
ignore)
else:
shutil.copyfile(src, dest)
In Python 3.8 the dirs_exist_ok keyword argument was added to shutil.copytree():
dirs_exist_okdictates whether to raise an exception in casedstor any missing parent directory already exists.
So, the following will work in recent versions of Python, even if the destination directory already exists:
shutil.copytree(src, dest, dirs_exist_ok=True) # 3.8+ only!
One major benefit is that it’s more flexible than distutils.dir_util.copy_tree(), as it takes additional arguments on files to ignore, etc. (see the documentation). On top of that, the accepted PEP 632 also states that distutils will be deprecated and subsequently removed in future versions of Python 3.
My simple answer.
def get_files_tree(src="src_path"):
req_files = []
for r, d, files in os.walk(src):
for file in files:
src_file = os.path.join(r, file)
src_file = src_file.replace('\', '/')
if src_file.endswith('.db'):
continue
req_files.append(src_file)
return req_files
def copy_tree_force(src_path="",dest_path=""):
"""
make sure that all the paths has correct slash characters.
"""
for cf in get_files_tree(src=src_path):
df= cf.replace(src_path, dest_path)
if not os.path.exists(os.path.dirname(df)):
os.makedirs(os.path.dirname(df))
shutil.copy2(cf, df)
There has been an update to shutil.copytree(). You can add the following argument:
shutil.copytree(src, dst, dirs_exist_ok=True)