Using python difflib to compare more than two files
Question:
I would like to get an overview over e.g. the ldd dependency list of multiple (3+) computers by comparing them with each other and highlighting the differences. For example, if I have a dict that looks as following:
my_ldd_outputs = {
01:"<ldd_output>",
02:"<ldd_output>",
...
09:"<ldd_output>",
10:"<ldd_output>"
}
I would like the output to look something like
<identical line 1>
<identical line 2>
<identical line 3>
<differing line 4> (computer 01 02)
<differing line 4> (computer 04 05 06 07)
<differing line 4> (computer 08 09 10)
<identical line 5>
<identical line 6>
...
My first approach involved python difflib, where my idea was to first get to a datastructure where all the ldd_output
lists (just the result split with n
) from the abovementioned my_ldd_outputs
dictionary are the same length, and any missing line that exists in another ldd_output
string is added with a string. So if two files looked like this:
ldd_1 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<extra line 5>
<identical line 6>
"""
ldd_2 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<identical line 6>
"""
My goal was to store those files as
ldd_1 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<extra line 5>
<identical line 6>
"""
ldd_2 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<None>
<identical line 6>
"""
And ultimately just iterate over every line of the converted files (which now all have the same length) and compare each line in terms of their differences and ignore any <None>
entries so the diff can be printed consecutively.
I created a function that uses python difflib
to fill the missing lines from other files with a <None>
string. However, I am not sure how to expand this function to incorporate an arbitrary amount of diffs
def generate_diff(file_1, file_2):
#differing hashvalues from ldd can be ignored, we only care about version and path
def remove_hashvalues(input):
return re.sub("([a-zA-Z0-9_.-]{32}/|([a-zA-Z0-9_.-]*))", "<>", input)
diff = [line.strip() for line in difflib.ndiff(remove_hashvalues(base).splitlines(keepends=True),remove_hashvalues(file_2).splitlines(keepends=True))]
list_1 = []
list_2 = []
i = 0
while i<len(diff):
if diff[i].strip():
if diff[i][0:2]=="- ":
lost = []
gained = []
while diff[i][0:2]=="- " or diff[i][0:2]=="? ":
if diff[i][0:2]=="- ": lost.append(diff[i][1:].strip())
i+=1
while diff[i][0:2]=="+ " or diff[i][0:2]=="? ":
if diff[i][0:2]=="+ ": gained.append(diff[i][1:].strip())
i+=1
while len(lost) != len(gained):
lost.append("<None>") if len(lost)<len(gained) else gained.insert(0,"<None>")
list_1+=lost; list_2+=gained
elif diff[i][0:2]=="+ ":
list_1.append("<None>"); list_2.append(diff[i][1:].strip())
if not diff[i][0:2]=="? ":
list_1.append(diff[i].strip()); list_2.append(diff[i].strip())
i+=1
return list_1, list_2
I also found this tool that allows the comparison of multiple files, but unfortunately its not designed to compare code.
EDIT: I adjusted the solution suggestion of @AyoubKaanich to create a more simplified version that does what I want:
from collections import defaultdict
import re
def transform(input):
input = re.sub("([a-zA-Z0-9_.-]{32}/|([a-zA-Z0-9_.-]*))", "<>", input) # differing hashvalues can be ignored, we only care about version and path
return sorted(input.splitlines())
def generate_diff(outputs: dict):
mapping = defaultdict(set)
for target, output in outputs.items():
for line in transform(output):
mapping[line.strip()].add(target)
result = []
current_line = None
color_index = 0
for line in sorted(mapping.keys()):
if len(outputs) == len(mapping[line]):
if current_line: current_line = None
result.append((line))
else:
if current_line != line.split(" ")[0]:
current_line = line.split(" ")[0]
color_index+=1
result.append((f"
I would like to get an overview over e.g. the ldd dependency list of multiple (3+) computers by comparing them with each other and highlighting the differences. For example, if I have a dict that looks as following:
my_ldd_outputs = {
01:"<ldd_output>",
02:"<ldd_output>",
...
09:"<ldd_output>",
10:"<ldd_output>"
}
I would like the output to look something like
<identical line 1>
<identical line 2>
<identical line 3>
<differing line 4> (computer 01 02)
<differing line 4> (computer 04 05 06 07)
<differing line 4> (computer 08 09 10)
<identical line 5>
<identical line 6>
...
My first approach involved python difflib, where my idea was to first get to a datastructure where all the ldd_output
lists (just the result split with n
) from the abovementioned my_ldd_outputs
dictionary are the same length, and any missing line that exists in another ldd_output
string is added with a string. So if two files looked like this:
ldd_1 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<extra line 5>
<identical line 6>
"""
ldd_2 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<identical line 6>
"""
My goal was to store those files as
ldd_1 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<extra line 5>
<identical line 6>
"""
ldd_2 = """
<identical line 1>
<identical line 2>
<differing line 3>
<identical line 4>
<None>
<identical line 6>
"""
And ultimately just iterate over every line of the converted files (which now all have the same length) and compare each line in terms of their differences and ignore any <None>
entries so the diff can be printed consecutively.
I created a function that uses python difflib
to fill the missing lines from other files with a <None>
string. However, I am not sure how to expand this function to incorporate an arbitrary amount of diffs
def generate_diff(file_1, file_2):
#differing hashvalues from ldd can be ignored, we only care about version and path
def remove_hashvalues(input):
return re.sub("([a-zA-Z0-9_.-]{32}/|([a-zA-Z0-9_.-]*))", "<>", input)
diff = [line.strip() for line in difflib.ndiff(remove_hashvalues(base).splitlines(keepends=True),remove_hashvalues(file_2).splitlines(keepends=True))]
list_1 = []
list_2 = []
i = 0
while i<len(diff):
if diff[i].strip():
if diff[i][0:2]=="- ":
lost = []
gained = []
while diff[i][0:2]=="- " or diff[i][0:2]=="? ":
if diff[i][0:2]=="- ": lost.append(diff[i][1:].strip())
i+=1
while diff[i][0:2]=="+ " or diff[i][0:2]=="? ":
if diff[i][0:2]=="+ ": gained.append(diff[i][1:].strip())
i+=1
while len(lost) != len(gained):
lost.append("<None>") if len(lost)<len(gained) else gained.insert(0,"<None>")
list_1+=lost; list_2+=gained
elif diff[i][0:2]=="+ ":
list_1.append("<None>"); list_2.append(diff[i][1:].strip())
if not diff[i][0:2]=="? ":
list_1.append(diff[i].strip()); list_2.append(diff[i].strip())
i+=1
return list_1, list_2
I also found this tool that allows the comparison of multiple files, but unfortunately its not designed to compare code.
EDIT: I adjusted the solution suggestion of @AyoubKaanich to create a more simplified version that does what I want:
from collections import defaultdict
import re
def transform(input):
input = re.sub("([a-zA-Z0-9_.-]{32}/|([a-zA-Z0-9_.-]*))", "<>", input) # differing hashvalues can be ignored, we only care about version and path
return sorted(input.splitlines())
def generate_diff(outputs: dict):
mapping = defaultdict(set)
for target, output in outputs.items():
for line in transform(output):
mapping[line.strip()].add(target)
result = []
current_line = None
color_index = 0
for line in sorted(mapping.keys()):
if len(outputs) == len(mapping[line]):
if current_line: current_line = None
result.append((line))
else:
if current_line != line.split(" ")[0]:
current_line = line.split(" ")[0]
color_index+=1
result.append((f"