Errorless Indenting™

This commit introduces the concept of Errorless Indenting™. From now on,
there will be no more `syntax error on line X` messages. The template will
be indented no matter what.

The reason that errors aren't fatal anymore is that the improved parsing
algorithm now gives correct output when Django template tags and HTML tags
are mixed in seemingly paradoxal ways. In almost all cases DjHTML indents
the template exactly how I would have done it manually. Which is really the
goal of this program, after all.

For the remaining cases, the new `{# fmt:off #}` tag disables formatting,
similar to Black.

Closes #17

djhtml/__main__.py

@@ -41,8 +41,7 @@ def main():
             " HTML/CSS/Javascript templates. It works similar to other"
             " code-formatting tools such as Black. The goal is to correctly"
             " indent already well-structured templates but not to fix broken"
-            " ones. A non-zero exit status indicates that a template could not"
-            " be indented."
+            " ones."
         ),
     )
     parser.add_argument(
@@ -77,7 +76,11 @@ def main():
         sys.exit("Will not modify files in-place without -i option")
 
     for input_file in args.input_files:
-        source = input_file.read()
+        try:
+            source = input_file.read()
+        except Exception:
+            print(f"\nFatal error while processing {input_file.name}\n")
+            raise
 
         try:
             if args.debug:
@@ -93,6 +96,14 @@ def main():
                 )
             exit_status = 1
             continue
+        except Exception:
+            print(
+                f"\nFatal error while processing {input_file.name}\n\n"
+                "    If you have time and are using the latest version, we\n"
+                "    would very much appreciate if you opened an issue on\n"
+                "    https://github.com/rtts/djhtml/issues\n"
+            )
+            raise
         finally:
             input_file.close()
 

djhtml/lines.py

@@ -4,8 +4,8 @@ class Line:
 
     """
 
-    def __init__(self, line_nr=1):
-        self.line_nr = line_nr
+    def __init__(self, nr=1):
+        self.nr = nr
         self.tokens = []
         self.level = 0
 
@@ -14,7 +14,7 @@ def append(self, token):
         Append tokens to the line.
 
         """
-        token.line_nr = self.line_nr
+        token.line_nr = self.nr
         self.tokens.append(token)
 
     @property
@@ -47,4 +47,4 @@ def __bool__(self):
         return bool(self.tokens and self.text)
 
     def __next__(self):
-        return Line(line_nr=self.line_nr + 1)
+        return Line(nr=self.nr + 1)

djhtml/modes.py

@@ -53,27 +53,33 @@ def parse(self):
                 opening_token = None
 
                 # When a dedenting token is found, match it with the
-                # token at the top of the stack. If there is no match,
-                # raise a syntax error.
+                # token at the top of the stack.
                 if token.dedents:
                     try:
-                        opening_token = stack.pop()
-                        assert token.kind == opening_token.kind
-                    except (IndexError, AssertionError):
-                        raise SyntaxError(
-                            f"illegal closing “{token.text}” on line {line.line_nr}"
-                        )
+                        if stack[-1].kind == token.kind:
+                            opening_token = stack.pop()
+                        elif token.kind == "django":
+                            opening_token = stack.pop()
+                            while opening_token.kind != token.kind:
+                                opening_token = stack.pop()
+                        elif first_token:
+                            # This closing token could not be matched.
+                            # Instead of erroring out, set the line level
+                            # to what it would have been with a
+                            # regular text token.
+                            line.level = stack[-1].level + 1
+                    except IndexError:
+                        line.level = 0
 
                     # If this dedenting token is the first in line,
-                    # it's somewhat special: the line level will be
-                    # set to to the line level of the corresponding
-                    # opening token.
-                    if first_token:
+                    # set the line level to the line level of the
+                    # corresponding opening token.
+                    if first_token and opening_token:
                         line.level = opening_token.level
 
                 # If the first token is not a dedenting token, the
-                # line level will one higher than that of the token at
-                # the top of the stack.
+                # line level will be one higher than that of the token
+                # at the top of the stack.
                 elif first_token:
                     line.level = stack[-1].level + 1 if stack else 0
 
@@ -89,11 +95,6 @@ def parse(self):
                 if token.text.strip():
                     first_token = False
 
-        # Ensure the stack is empty at the end of the run.
-        if stack:
-            token = stack.pop()
-            raise SyntaxError(f"unclosed “{token.text}” on line {token.line_nr}")
-
     def tokenize(self):
         """
         Split the source text into tokens and place them on lines.
@@ -106,8 +107,8 @@ def tokenize(self):
 
         while True:
             try:
-                # Split the source at the first instance of one of the
-                # current mode's raw tokens.
+                # Split the source at the first occurence of one of
+                # the current mode's raw tokens.
                 head, raw_token, tail = mode.token_re.split(src, maxsplit=1)
 
             except ValueError:
@@ -160,6 +161,12 @@ def create_token(self, raw_token, src):
             elif name.startswith("end"):
                 token = Token.Close(raw_token, kind)
 
+        elif tag := re.match(r"{# *(\w+:\w+).*?#}", raw_token):
+            name = tag.group(1)
+            if name == "fmt:off":
+                token = Token.Open(raw_token, kind)
+                self.next_mode = Comment(r"\{% *fmt:on.*?%\}", self, kind)
+
         return token
 
     def debug(self):
@@ -188,7 +195,6 @@ class DjHTML(DjTXT):
     ]
 
     IGNORE_TAGS = [
-        "doctype",
         "area",
         "base",
         "br",
@@ -308,7 +314,7 @@ def create_token(self, raw_token, src):
         return super().create_token(raw_token, src)
 
 
-# The following are "special" modes with slightly different constructors.
+# The following are "special" modes with different constructors.
 
 
 class Comment(DjTXT):

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = djhtml
-version = 1.3.0
+version = 1.4.0
 description = Django template indenter
 long_description = file: README.md
 long_description_content_type = text/markdown

tests/suite/paradoxes.in

@@ -0,0 +1,74 @@
+<!-- Common pattern in django templates -->
+{% if %}
+<a>
+{% else %}
+<a>
+{% endif %}
+.
+{% if %}
+</a>
+{% endif %}
+
+
+<!-- The literal example from issue #17 -->
+{% if some_condition %}
+    <a href="{% url some_url">
+{% elif some_other_condition %}
+    <a href="{% url some_other_url">
+{% endif %}
+    [a bunch of details go here]
+{% if end_tag_needed %}</a>{% endif %}
+
+
+<!-- Deep nestings should be instantly dedented -->
+{% block %}
+<a>
+<a>
+<a>
+<a>
+<a>
+{% endblock %}
+{% for x in [1,2,3,4,5] %}
+</a>
+{% endfor %}
+
+
+<!-- The most complicated example I can think of -->
+<body>
+{% if %}
+<div>
+{% else %}
+<span>{% if %}<blink>
+{% block %}
+</blink>
+{% endblock %}{% endif %}
+{% endif %}
+.
+{% if %}
+</span>
+{% else %}
+</div>
+{% endif %}
+</body>
+
+
+<!-- Inside <script> tag -->
+<script>
+{% if jquery %}
+$(function() {
+{% else %}
+document.addEventListener("DOMContentLoaded", function(event) {
+{% endif %}
+console.log("Time to ditch jQuery!")
+});
+</script>
+
+
+<!-- Manually disable formatting -->
+<body>
+    {# fmt:off #}
+     ,-._|\
+    /     .\
+    \_,--._/
+    {# fmt:on #}
+</body>

tests/suite/paradoxes.out

@@ -0,0 +1,74 @@
+<!-- Common pattern in django templates -->
+{% if %}
+    <a>
+{% else %}
+    <a>
+{% endif %}
+.
+{% if %}
+    </a>
+{% endif %}
+
+
+<!-- The literal example from issue #17 -->
+{% if some_condition %}
+    <a href="{% url some_url">
+{% elif some_other_condition %}
+    <a href="{% url some_other_url">
+{% endif %}
+[a bunch of details go here]
+{% if end_tag_needed %}</a>{% endif %}
+
+
+<!-- Deep nestings should be instantly dedented -->
+{% block %}
+    <a>
+        <a>
+            <a>
+                <a>
+                    <a>
+{% endblock %}
+{% for x in [1,2,3,4,5] %}
+    </a>
+{% endfor %}
+
+
+<!-- The most complicated example I can think of -->
+<body>
+    {% if %}
+        <div>
+    {% else %}
+        <span>{% if %}<blink>
+            {% block %}
+                </blink>
+            {% endblock %}{% endif %}
+    {% endif %}
+    .
+    {% if %}
+        </span>
+    {% else %}
+        </div>
+    {% endif %}
+</body>
+
+
+<!-- Inside <script> tag -->
+<script>
+    {% if jquery %}
+        $(function() {
+    {% else %}
+        document.addEventListener("DOMContentLoaded", function(event) {
+    {% endif %}
+    console.log("Time to ditch jQuery!")
+    });
+</script>
+
+
+<!-- Manually disable formatting -->
+<body>
+    {# fmt:off #}
+     ,-._|\
+    /     .\
+    \_,--._/
+    {# fmt:on #}
+</body>