Recursively find and replace string in text files
Question:
I want to recursively search through a directory with subdirectories of text files and replace every occurrence of {$replace} within the files with the contents of a multi line string. How can this be achieved with Python?
So far all I have is the recursive code using os.walk to get a list of files that are required to be changed.
import os
import sys
fileList = []
rootdir = "C:\test"
for root, subFolders, files in os.walk(rootdir):
if subFolders != ".svn":
for file in files:
fileParts = file.split('.')
if len(fileParts) > 1:
if fileParts[1] == "php":
fileList.append(os.path.join(root,file))
print fileList
Answers:
Check out os.walk:
import os
replacement = """some
multi-line string"""
for dname, dirs, files in os.walk("some_dir"):
for fname in files:
fpath = os.path.join(dname, fname)
with open(fpath) as f:
s = f.read()
s = s.replace("{$replace}", replacement)
with open(fpath, "w") as f:
f.write(s)
The above solution has flaws, such as the fact that it opens literally every file it finds, or the fact that each file is read entirely into memory (which would be bad if you had a 1GB text file), but it should be a good starting point.
You also may want to look into the re module if you want to do a more complex find/replace than looking for a specific string.
os.walk is great. However, it looks like you need to filer file types (which I would suggest if you are going to walk some directory). To do this, you should add import fnmatch.
import os, fnmatch
def findReplace(directory, find, replace, filePattern):
for path, dirs, files in os.walk(os.path.abspath(directory)):
for filename in fnmatch.filter(files, filePattern):
filepath = os.path.join(path, filename)
with open(filepath) as f:
s = f.read()
s = s.replace(find, replace)
with open(filepath, "w") as f:
f.write(s)
This allows you to do something like:
findReplace("some_dir", "find this", "replace with this", "*.txt")
Here’s my code (which I think is the same as the above but I’m including it just in case there’s something subtly different about it):
import os, fnmatch, sys
def findReplace(directory, find, replace, filePattern):
for path, dirs, files in os.walk(os.path.abspath(directory)):
for filename in fnmatch.filter(files, filePattern):
filepath = os.path.join(path, filename)
with open(filepath) as f:
s = f.read()
s = s.replace(find, replace)
with open(filepath, "w") as f:
f.write(s)
it runs without error.
BUT, the file, in z:test is unchanged.
I’ve put in print statements, like print("got here") but they don’t print out either.
To avoid recursing into .svn directories, os.walk() allows you to change the dirs list inplace. To simplify the text replacement in a file without requiring to read the whole file in memory, you could use fileinput module. And to filter filenames using a file pattern, you could use fnmatch module as suggested by @David Sulpy:
#!/usr/bin/env python
from __future__ import print_function
import fnmatch
import os
from fileinput import FileInput
def find_replace(topdir, file_pattern, text, replacement):
for dirpath, dirs, files in os.walk(topdir, topdown=True):
dirs[:] = [d for d in dirs if d != '.svn'] # skip .svn dirs
files = [os.path.join(dirpath, filename)
for filename in fnmatch.filter(files, file_pattern)]
for line in FileInput(files, inplace=True):
print(line.replace(text, replacement), end='')
find_replace(r"C:test", "*.php", '{$replace}', "multilinenreplacement")
Sulpy’s answer is good but incomplete. The user would be likely to want to input the parameters through an entry widget, so we might have something more like this (also incomplete, but left as an exercise):
import os, fnmatch
from Tkinter import *
fields = 'Folder', 'Search', 'Replace', 'FilePattern'
def fetch(entvals):
# print entvals
# print ents
entItems = entvals.items()
for entItem in entItems:
field = entItem[0]
text = entItem[1].get()
print('%s: "%s"' % (field, text))
def findReplace(entvals):
# print ents
directory = entvals.get("Folder").get()
find = entvals.get("Search").get()
replace = entvals.get("Replace").get()
filePattern = entvals.get("FilePattern").get()
for path, dirs, files in os.walk(os.path.abspath(directory)):
for filename in fnmatch.filter(files, filePattern):
# print filename
filepath = os.path.join(path, filename)
print filepath # Can be commented out -- used for confirmation
with open(filepath) as f:
s = f.read()
s = s.replace(find, replace)
with open(filepath, "w") as f:
f.write(s)
def makeform(root, fields):
entvals = {}
for field in fields:
row = Frame(root)
lab = Label(row, width=17, text=field+": ", anchor='w')
ent = Entry(row)
row.pack(side=TOP, fill=X, padx=5, pady=5)
lab.pack(side=LEFT)
ent.pack(side=RIGHT, expand=YES, fill=X)
entvals[field] = ent
# print ent
return entvals
if __name__ == '__main__':
root = Tk()
root.title("Recursive S&R")
ents = makeform(root, fields)
# print ents
root.bind('<Return>', (lambda event, e=ents: fetch(e)))
b1 = Button(root, text='Show', command=(lambda e=ents: fetch(e)))
b1.pack(side=LEFT, padx=5, pady=5)
b2 = Button(root, text='Execute', command=(lambda e=ents: findReplace(e)))
b2.pack(side=LEFT, padx=5, pady=5)
b3 = Button(root, text='Quit', command=root.quit)
b3.pack(side=LEFT, padx=5, pady=5)
root.mainloop()
Multiple files string change
import glob
for allfiles in glob.glob(‘*.txt’):
for line in open(allfiles,'r'):
change=line.replace("old_string","new_string")
output=open(allfiles,'w')
output.write(change)
For those using Python 3.5+ you can now use a glob recursively with the use of ** and the recursive flag.
Here’s an example replacing hello with world for all .txt files:
for filepath in glob.iglob('./**/*.txt', recursive=True):
with open(filepath) as file:
s = file.read()
s = s.replace('hello', 'world')
with open(filepath, "w") as file:
file.write(s)
How about just using:
clean = ''.join([e for e in text if e != 'string'])
Use:
pip3 install manip
This lets you use a decorator to create something like:
@manip(at='.php$', recursive=True) # to apply to subfolders
def replace_on_php(text, find, replacement):
return text.replace(find, replacement)
Now in your prompt you should be able to call
replace_on_php('explode', 'myCustomExplode', path='./myPhPFiles', modify=True)
and this should make the function apply itself on the entire folder.
This is an old question but I figured I’d provide an updated and simpler answer using current libraries in python3.8.
from pathlib import Path
import re
rootdir = Path("C:\test")
pattern = r'REGEX for the text you want to replace'
replace = r'REGEX for what to replace it with'
for file in [ f for f in rootdir.glob("**.php") ]: #modify glob pattern as needed
file_contents = file.read_text()
new_file_contents = re.sub(f"{pattern}", f"{replace}", file_contents)
file.write_text(new_file_contents)
I want to recursively search through a directory with subdirectories of text files and replace every occurrence of {$replace} within the files with the contents of a multi line string. How can this be achieved with Python?
So far all I have is the recursive code using os.walk to get a list of files that are required to be changed.
import os
import sys
fileList = []
rootdir = "C:\test"
for root, subFolders, files in os.walk(rootdir):
if subFolders != ".svn":
for file in files:
fileParts = file.split('.')
if len(fileParts) > 1:
if fileParts[1] == "php":
fileList.append(os.path.join(root,file))
print fileList
Check out os.walk:
import os
replacement = """some
multi-line string"""
for dname, dirs, files in os.walk("some_dir"):
for fname in files:
fpath = os.path.join(dname, fname)
with open(fpath) as f:
s = f.read()
s = s.replace("{$replace}", replacement)
with open(fpath, "w") as f:
f.write(s)
The above solution has flaws, such as the fact that it opens literally every file it finds, or the fact that each file is read entirely into memory (which would be bad if you had a 1GB text file), but it should be a good starting point.
You also may want to look into the re module if you want to do a more complex find/replace than looking for a specific string.
os.walk is great. However, it looks like you need to filer file types (which I would suggest if you are going to walk some directory). To do this, you should add import fnmatch.
import os, fnmatch
def findReplace(directory, find, replace, filePattern):
for path, dirs, files in os.walk(os.path.abspath(directory)):
for filename in fnmatch.filter(files, filePattern):
filepath = os.path.join(path, filename)
with open(filepath) as f:
s = f.read()
s = s.replace(find, replace)
with open(filepath, "w") as f:
f.write(s)
This allows you to do something like:
findReplace("some_dir", "find this", "replace with this", "*.txt")
Here’s my code (which I think is the same as the above but I’m including it just in case there’s something subtly different about it):
import os, fnmatch, sys
def findReplace(directory, find, replace, filePattern):
for path, dirs, files in os.walk(os.path.abspath(directory)):
for filename in fnmatch.filter(files, filePattern):
filepath = os.path.join(path, filename)
with open(filepath) as f:
s = f.read()
s = s.replace(find, replace)
with open(filepath, "w") as f:
f.write(s)
it runs without error.
BUT, the file, in z:test is unchanged.
I’ve put in print statements, like print("got here") but they don’t print out either.
To avoid recursing into .svn directories, os.walk() allows you to change the dirs list inplace. To simplify the text replacement in a file without requiring to read the whole file in memory, you could use fileinput module. And to filter filenames using a file pattern, you could use fnmatch module as suggested by @David Sulpy:
#!/usr/bin/env python
from __future__ import print_function
import fnmatch
import os
from fileinput import FileInput
def find_replace(topdir, file_pattern, text, replacement):
for dirpath, dirs, files in os.walk(topdir, topdown=True):
dirs[:] = [d for d in dirs if d != '.svn'] # skip .svn dirs
files = [os.path.join(dirpath, filename)
for filename in fnmatch.filter(files, file_pattern)]
for line in FileInput(files, inplace=True):
print(line.replace(text, replacement), end='')
find_replace(r"C:test", "*.php", '{$replace}', "multilinenreplacement")
Sulpy’s answer is good but incomplete. The user would be likely to want to input the parameters through an entry widget, so we might have something more like this (also incomplete, but left as an exercise):
import os, fnmatch
from Tkinter import *
fields = 'Folder', 'Search', 'Replace', 'FilePattern'
def fetch(entvals):
# print entvals
# print ents
entItems = entvals.items()
for entItem in entItems:
field = entItem[0]
text = entItem[1].get()
print('%s: "%s"' % (field, text))
def findReplace(entvals):
# print ents
directory = entvals.get("Folder").get()
find = entvals.get("Search").get()
replace = entvals.get("Replace").get()
filePattern = entvals.get("FilePattern").get()
for path, dirs, files in os.walk(os.path.abspath(directory)):
for filename in fnmatch.filter(files, filePattern):
# print filename
filepath = os.path.join(path, filename)
print filepath # Can be commented out -- used for confirmation
with open(filepath) as f:
s = f.read()
s = s.replace(find, replace)
with open(filepath, "w") as f:
f.write(s)
def makeform(root, fields):
entvals = {}
for field in fields:
row = Frame(root)
lab = Label(row, width=17, text=field+": ", anchor='w')
ent = Entry(row)
row.pack(side=TOP, fill=X, padx=5, pady=5)
lab.pack(side=LEFT)
ent.pack(side=RIGHT, expand=YES, fill=X)
entvals[field] = ent
# print ent
return entvals
if __name__ == '__main__':
root = Tk()
root.title("Recursive S&R")
ents = makeform(root, fields)
# print ents
root.bind('<Return>', (lambda event, e=ents: fetch(e)))
b1 = Button(root, text='Show', command=(lambda e=ents: fetch(e)))
b1.pack(side=LEFT, padx=5, pady=5)
b2 = Button(root, text='Execute', command=(lambda e=ents: findReplace(e)))
b2.pack(side=LEFT, padx=5, pady=5)
b3 = Button(root, text='Quit', command=root.quit)
b3.pack(side=LEFT, padx=5, pady=5)
root.mainloop()
Multiple files string change
import glob
for allfiles in glob.glob(‘*.txt’):
for line in open(allfiles,'r'):
change=line.replace("old_string","new_string")
output=open(allfiles,'w')
output.write(change)
For those using Python 3.5+ you can now use a glob recursively with the use of ** and the recursive flag.
Here’s an example replacing hello with world for all .txt files:
for filepath in glob.iglob('./**/*.txt', recursive=True):
with open(filepath) as file:
s = file.read()
s = s.replace('hello', 'world')
with open(filepath, "w") as file:
file.write(s)
How about just using:
clean = ''.join([e for e in text if e != 'string'])
Use:
pip3 install manip
This lets you use a decorator to create something like:
@manip(at='.php$', recursive=True) # to apply to subfolders
def replace_on_php(text, find, replacement):
return text.replace(find, replacement)
Now in your prompt you should be able to call
replace_on_php('explode', 'myCustomExplode', path='./myPhPFiles', modify=True)
and this should make the function apply itself on the entire folder.
This is an old question but I figured I’d provide an updated and simpler answer using current libraries in python3.8.
from pathlib import Path
import re
rootdir = Path("C:\test")
pattern = r'REGEX for the text you want to replace'
replace = r'REGEX for what to replace it with'
for file in [ f for f in rootdir.glob("**.php") ]: #modify glob pattern as needed
file_contents = file.read_text()
new_file_contents = re.sub(f"{pattern}", f"{replace}", file_contents)
file.write_text(new_file_contents)