How to Pretty Print HTML to a file, with indentation
Question:
I am using lxml.html to generate some HTML. I want to pretty print (with indentation) my final result into an html file. How do I do that?
This is what I have tried and got till now
import lxml.html as lh
from lxml.html import builder as E
sliderRoot=lh.Element("div", E.CLASS("scroll"), style="overflow-x: hidden; overflow-y: hidden;")
scrollContainer=lh.Element("div", E.CLASS("scrollContainer"), style="width: 4340px;")
sliderRoot.append(scrollContainer)
print lh.tostring(sliderRoot, pretty_print = True, method="html")
As you can see I am using the pretty_print=True attribute. I thought that would give indented code, but it doesn’t really help. This is the output :
<div style="overflow-x: hidden; overflow-y: hidden;" class="scroll"><div style="width: 4340px;" class="scrollContainer"></div></div>
Answers:
If you don’t care about quirky HTMLness (e.g. you must support absolutely support those hordes of Netscpae 2.0-using clients, so having <br> instead of <br /> is a must), you can always change your method to “xml”, which seems to work. This is probably a bug in lxml or in libxml, but I couldn’t find the reason for it.
Couldn’t you just pipe it into HTML Tidy? Either from the shell or through os.system().
not really my code, I picked it somewhere
def indent(elem, level=0):
i = 'n' + level * ' '
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + ' '
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
I use it with:
indent(page)
tostring(page)
Under the hood, lxml uses libxml2 to serialize the tree back into a string. Here is the relevant snippet of code that determines whether to append a newline after closing a tag:
xmlOutputBufferWriteString(buf, ">");
if ((format) && (!info->isinline) && (cur->next != NULL)) {
if ((cur->next->type != HTML_TEXT_NODE) &&
(cur->next->type != HTML_ENTITY_REF_NODE) &&
(cur->parent != NULL) &&
(cur->parent->name != NULL) &&
(cur->parent->name[0] != 'p')) /* p, pre, param */
xmlOutputBufferWriteString(buf, "n");
}
return;
So if a node is an element, is not an inline tag and is followed by a sibling node (cur->next != NULL) and isn’t one of p, pre, param then it will output a newline.
I ended up using BeautifulSoup directly. That is something lxml.html.soupparser uses for parsing HTML.
BeautifulSoup has a prettify method that does exactly what it says it does. It prettifies the HTML with proper indents and everything.
BeautifulSoup will NOT fix the HTML, so broken code, remains broken. But in this case, since the code is being generated by lxml, the HTML code should be at least semantically correct.
In the example given in my question, I will have to do this :
from bs4 import BeautifulSoup as bs
root = lh.tostring(sliderRoot) #convert the generated HTML to a string
soup = bs(root) #make BeautifulSoup
prettyHTML = soup.prettify() #prettify the html
Though my answer might not be helpful now, I am dropping it here to act as a reference to anybody else in future.
lxml.html.tostring(), indeed, doesn’t pretty print the provided HTML in spite of pretty_print=True.
However, the "sibling" of lxml.html – lxml.etree has it working well.
So one might use it as following:
from lxml import etree, html
document_root = html.fromstring("<html><body><h1>hello world</h1></body></html>")
print(etree.tostring(document_root, encoding='unicode', pretty_print=True))
The output is like this:
<html>
<body>
<h1>hello world</h1>
</body>
</html>
If you store the HTML as an unformatted string, in a variable html_string, it can be done using beautifulsoup4 as follows:
from bs4 import BeautifulSoup
print(BeautifulSoup(html_string, 'html.parser').prettify())
If adding one more dependency is not a problem, you can use the html5print package. The advantage over the other solutions, is that it also beautifies both CSS and Javascript code embedded in the HTML document.
To install it, execute:
pip install html5print
Then, you can either use it as a command:
html5-print ugly.html -o pretty.html
or as Python code:
from html5print import HTMLBeautifier
html = '<title>Page Title</title><p>Some text here</p>'
print(HTMLBeautifier.beautify(html, 4))
I tried both BeautifulSoup’s prettify and html5print’s HTMLBeautifier solutions but since I’m using yattag to generate HTML it seems more appropriate to use its indent function, which produces nicely indented output.
from yattag import indent
rawhtml = "String with some HTML code..."
result = indent(
rawhtml,
indentation = ' ',
newline = 'rn',
indent_text = True
)
print(result)
This is crude and not very robust but it will balance out the example html string, without using any non-standard libraries
import re
html = """
<A value="X"><B value="X">
<A value="X"><B value="X">
some random text
</B></A><C />
some random text
</B></A><C />
"""
rx_al = r"(<[/A-Za-z]+[^>]*>)"
rx_op = r"<([A-Za-z]+)[^/]+>"
rx_cl = r"</([A-Za-z]+)>"
# self-closing not needed
#rx_sc = r"<([A-Za-z]+).* />"
matches = re.findall(rx_al, html, flags=re.M)
def lookup_key(match, indent):
return f"{match[0]}:{indent}"
def balance(nodes):
builder = []
indent = 0
lookup = {}
for node in nodes:
is_open = re.match(rx_op, node)
is_close = re.match(rx_cl, node)
padl = " " * indent
if is_open:
k = lookup_key(is_open, indent)
lookup[k] = node
indent += 2
elif is_close:
for i in range(0, indent, -2):
if lookup.pop(lookup_key(is_close, i), None):
break
indent -= 2
padl = padl[:-2]
builder.append(padl + node)
return "n".join(builder)
print(balance(matches))
will produce:
<A value="X">
<B value="X">
<A value="X">
<B value="X">
</B>
</A>
<C />
</B>
</A>
<C />
Customize indented tags and text (by default, etree.indent cannot indent text).
from lxml import etree
import textwrap
def etree_pretty(root, space="t"):
for elem in root.iterdescendants():
if elem.text:
depth = int(elem.xpath("count(ancestor::*)")) + 1
temp_text = textwrap.dedent(elem.text).strip()
elem.text = (
"n"
+ textwrap.indent(temp_text, prefix=depth * space)
+ ("n" + (depth-1) * space)
)
etree.indent(root, space=space)
html_str = """
<html><body><h1>hello world</h1><h2>sub title</h2></body></html>
"""
root = etree.HTML(html_str)
etree_pretty(root)
print(etree.tostring(root, encoding="unicode"))
Print:
<html>
<body>
<h1>
hello world
</h1>
<h2>
sub title
</h2>
</body>
</html>
I am using lxml.html to generate some HTML. I want to pretty print (with indentation) my final result into an html file. How do I do that?
This is what I have tried and got till now
import lxml.html as lh
from lxml.html import builder as E
sliderRoot=lh.Element("div", E.CLASS("scroll"), style="overflow-x: hidden; overflow-y: hidden;")
scrollContainer=lh.Element("div", E.CLASS("scrollContainer"), style="width: 4340px;")
sliderRoot.append(scrollContainer)
print lh.tostring(sliderRoot, pretty_print = True, method="html")
As you can see I am using the pretty_print=True attribute. I thought that would give indented code, but it doesn’t really help. This is the output :
<div style="overflow-x: hidden; overflow-y: hidden;" class="scroll"><div style="width: 4340px;" class="scrollContainer"></div></div>
If you don’t care about quirky HTMLness (e.g. you must support absolutely support those hordes of Netscpae 2.0-using clients, so having <br> instead of <br /> is a must), you can always change your method to “xml”, which seems to work. This is probably a bug in lxml or in libxml, but I couldn’t find the reason for it.
Couldn’t you just pipe it into HTML Tidy? Either from the shell or through os.system().
not really my code, I picked it somewhere
def indent(elem, level=0):
i = 'n' + level * ' '
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + ' '
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
I use it with:
indent(page)
tostring(page)
Under the hood, lxml uses libxml2 to serialize the tree back into a string. Here is the relevant snippet of code that determines whether to append a newline after closing a tag:
xmlOutputBufferWriteString(buf, ">");
if ((format) && (!info->isinline) && (cur->next != NULL)) {
if ((cur->next->type != HTML_TEXT_NODE) &&
(cur->next->type != HTML_ENTITY_REF_NODE) &&
(cur->parent != NULL) &&
(cur->parent->name != NULL) &&
(cur->parent->name[0] != 'p')) /* p, pre, param */
xmlOutputBufferWriteString(buf, "n");
}
return;
So if a node is an element, is not an inline tag and is followed by a sibling node (cur->next != NULL) and isn’t one of p, pre, param then it will output a newline.
I ended up using BeautifulSoup directly. That is something lxml.html.soupparser uses for parsing HTML.
BeautifulSoup has a prettify method that does exactly what it says it does. It prettifies the HTML with proper indents and everything.
BeautifulSoup will NOT fix the HTML, so broken code, remains broken. But in this case, since the code is being generated by lxml, the HTML code should be at least semantically correct.
In the example given in my question, I will have to do this :
from bs4 import BeautifulSoup as bs
root = lh.tostring(sliderRoot) #convert the generated HTML to a string
soup = bs(root) #make BeautifulSoup
prettyHTML = soup.prettify() #prettify the html
Though my answer might not be helpful now, I am dropping it here to act as a reference to anybody else in future.
lxml.html.tostring(), indeed, doesn’t pretty print the provided HTML in spite of pretty_print=True.
However, the "sibling" of lxml.html – lxml.etree has it working well.
So one might use it as following:
from lxml import etree, html
document_root = html.fromstring("<html><body><h1>hello world</h1></body></html>")
print(etree.tostring(document_root, encoding='unicode', pretty_print=True))
The output is like this:
<html>
<body>
<h1>hello world</h1>
</body>
</html>
If you store the HTML as an unformatted string, in a variable html_string, it can be done using beautifulsoup4 as follows:
from bs4 import BeautifulSoup
print(BeautifulSoup(html_string, 'html.parser').prettify())
If adding one more dependency is not a problem, you can use the html5print package. The advantage over the other solutions, is that it also beautifies both CSS and Javascript code embedded in the HTML document.
To install it, execute:
pip install html5print
Then, you can either use it as a command:
html5-print ugly.html -o pretty.html
or as Python code:
from html5print import HTMLBeautifier
html = '<title>Page Title</title><p>Some text here</p>'
print(HTMLBeautifier.beautify(html, 4))
I tried both BeautifulSoup’s prettify and html5print’s HTMLBeautifier solutions but since I’m using yattag to generate HTML it seems more appropriate to use its indent function, which produces nicely indented output.
from yattag import indent
rawhtml = "String with some HTML code..."
result = indent(
rawhtml,
indentation = ' ',
newline = 'rn',
indent_text = True
)
print(result)
This is crude and not very robust but it will balance out the example html string, without using any non-standard libraries
import re
html = """
<A value="X"><B value="X">
<A value="X"><B value="X">
some random text
</B></A><C />
some random text
</B></A><C />
"""
rx_al = r"(<[/A-Za-z]+[^>]*>)"
rx_op = r"<([A-Za-z]+)[^/]+>"
rx_cl = r"</([A-Za-z]+)>"
# self-closing not needed
#rx_sc = r"<([A-Za-z]+).* />"
matches = re.findall(rx_al, html, flags=re.M)
def lookup_key(match, indent):
return f"{match[0]}:{indent}"
def balance(nodes):
builder = []
indent = 0
lookup = {}
for node in nodes:
is_open = re.match(rx_op, node)
is_close = re.match(rx_cl, node)
padl = " " * indent
if is_open:
k = lookup_key(is_open, indent)
lookup[k] = node
indent += 2
elif is_close:
for i in range(0, indent, -2):
if lookup.pop(lookup_key(is_close, i), None):
break
indent -= 2
padl = padl[:-2]
builder.append(padl + node)
return "n".join(builder)
print(balance(matches))
will produce:
<A value="X">
<B value="X">
<A value="X">
<B value="X">
</B>
</A>
<C />
</B>
</A>
<C />
Customize indented tags and text (by default, etree.indent cannot indent text).
from lxml import etree
import textwrap
def etree_pretty(root, space="t"):
for elem in root.iterdescendants():
if elem.text:
depth = int(elem.xpath("count(ancestor::*)")) + 1
temp_text = textwrap.dedent(elem.text).strip()
elem.text = (
"n"
+ textwrap.indent(temp_text, prefix=depth * space)
+ ("n" + (depth-1) * space)
)
etree.indent(root, space=space)
html_str = """
<html><body><h1>hello world</h1><h2>sub title</h2></body></html>
"""
root = etree.HTML(html_str)
etree_pretty(root)
print(etree.tostring(root, encoding="unicode"))
Print:
<html>
<body>
<h1>
hello world
</h1>
<h2>
sub title
</h2>
</body>
</html>