From c2c706d67ad44dd0f244239d6db30d22db4fd2f7 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Mon, 13 Feb 2023 18:04:17 +0100
Subject: [PATCH 01/45] DjHTML Version 3

For this release, a substantial part the codebase has been refactored.
Here is a non-exhaustive summary of all the things that have changed:

- Support for Python 3.6 and 3.7 has been dropped
- A Django middleware class to indent HTML responses has been added
- Ignore both opening and closing comment tags
- New handling off both relative and absolute offsets
- Return correct `repr()` strings with the `--debug` option
- New token class "OpenDouble" and a revised indentation algorithm
- Additional `line` argument to `create_token()` methods
- Changed return value of `create_token()` methods
- Changed constructor arguments of Token and Line classes
- Refactored all `create_token()` methods to get rid of return statements
- Simplified test suite runner that reindents the expected output
- Don't indent the contents of template variables (closes #75)
- Improved handling of Django tags inside HTML elements
- Improved JavaScript `case` indentation (closes #76)
- Improved JavaScript method chaining (closes #59)
- Improved CSS multiline statements (closes #74)
- New multiline HTML element indentation (closes #50)
- New multiline HTML attribute value indentation
- Extensive test coverage with lots of edge cases
---
 .github/workflows/main.yml                    |   4 +-
 README.md                                     |  52 ++-
 djhtml/__main__.py                            |   9 +-
 djhtml/lines.py                               |  48 +-
 djhtml/middleware.py                          |  35 ++
 djhtml/modes.py                               | 423 +++++++++---------
 djhtml/tokens.py                              |  32 +-
 noxfile.py                                    |   2 +-
 setup.cfg                                     |   9 +-
 tests/suite/css.html                          |  53 +++
 tests/suite/css.in                            |  38 --
 tests/suite/css.out                           |  38 --
 tests/suite/{django.out => django.html}       |  26 ++
 tests/suite/django.in                         | 112 -----
 tests/suite/html.html                         | 330 ++++++++++++++
 tests/suite/html.in                           | 173 -------
 tests/suite/html.out                          | 173 -------
 tests/suite/html_arrow_attr.in                |   5 -
 tests/suite/html_arrow_attr.out               |   5 -
 tests/suite/html_tag.in                       |  21 -
 tests/suite/html_tag.out                      |  21 -
 tests/suite/{jinja.out => jinja.html}         |   7 +
 tests/suite/jinja.in                          |  47 --
 tests/suite/{js.out => js.html}               |  39 ++
 tests/suite/js.in                             | 231 ----------
 tests/suite/{paradoxes.out => paradoxes.html} |  10 -
 tests/suite/paradoxes.in                      |  74 ---
 tests/test_suite.py                           |  49 +-
 28 files changed, 805 insertions(+), 1261 deletions(-)
 create mode 100644 djhtml/middleware.py
 create mode 100644 tests/suite/css.html
 delete mode 100644 tests/suite/css.in
 delete mode 100644 tests/suite/css.out
 rename tests/suite/{django.out => django.html} (84%)
 delete mode 100644 tests/suite/django.in
 create mode 100644 tests/suite/html.html
 delete mode 100644 tests/suite/html.in
 delete mode 100644 tests/suite/html.out
 delete mode 100644 tests/suite/html_arrow_attr.in
 delete mode 100644 tests/suite/html_arrow_attr.out
 delete mode 100644 tests/suite/html_tag.in
 delete mode 100644 tests/suite/html_tag.out
 rename tests/suite/{jinja.out => jinja.html} (84%)
 delete mode 100644 tests/suite/jinja.in
 rename tests/suite/{js.out => js.html} (84%)
 delete mode 100644 tests/suite/js.in
 rename tests/suite/{paradoxes.out => paradoxes.html} (90%)
 delete mode 100644 tests/suite/paradoxes.in

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4d80350..c1f874a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -9,13 +9,11 @@ on:
 jobs:
   tests:
     name: Python ${{ matrix.python-version }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
         python-version:
-        - 3.6
-        - 3.7
         - 3.8
         - 3.9
         - '3.10'
diff --git a/README.md b/README.md
index e6daacd..7d9b2a0 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,7 @@
 
 ***A pure-Python Django/Jinja template indenter without dependencies.***
 
-DjHTML is a fully automatic template indenter that works with mixed
-HTML/CSS/Javascript templates that contain
+DjHTML indents mixed HTML/CSS/Javascript templates that contain
 [Django](https://docs.djangoproject.com/en/stable/ref/templates/language/)
 or [Jinja](https://jinja.palletsprojects.com/templates/) template
 tags. It works similar to other code-formatting tools such as
@@ -59,10 +58,10 @@ Install DjHTML with the following command:
 
     $ pip install djhtml
 
-Note that [Windows still uses legacy code pages for the system
-encoding](https://docs.python.org/3/using/windows.html#win-utf8-mode).
-It is highly advised to set the environment variable `PYTHONUTF8` to
-`1` to avoid issues with indenting UTF-8 files. You can do so with the
+Note that
+[Windows still uses legacy code pages](https://docs.python.org/3/using/windows.html#win-utf8-mode)
+instead of UTF-8. It is highly advised to set the environment variable
+`PYTHONUTF8` to `1` with the
 [setx](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/setx)
 command:
 
@@ -78,12 +77,42 @@ command:
     reindented template.html
     1 template has been reindented.
 
+You can also run `djhtml .` to indent all HTML files beneath the
+current directory.
+
 An exit status of 0 means that everything went well, regardless of
 whether any files were changed. When the option `-c` / `--check` is
 used, the exit status is 1 when one or more files would have changed,
-but no changes are actually made. The exit status of 123 means that
-there was an error while indenting one or more files. All available
-options are given by `djthml -h` / `djthml --help`.
+but no changes are actually made. All available options are given by
+`djthml -h` / `djthml --help`.
+
+
+## New! Multi-line HTML tag alignment
+
+As of version 3, DjHTML indents multi-line HTML tags and multi-line
+attribute values like this:
+
+```html
+<blockquote cite="John Lennon"
+            style="color: dimgray;
+                   font-style: italic;
+                   border-left: 5px solid gray;
+                  ">
+    It's weird not to be weird.
+</blockquote>
+```
+
+
+## New! Django middleware
+
+To automatically indent all the HTML responses from your Django web
+application, add the following to your settings file:
+
+    TABWIDTH = 4
+    MIDDLEWARE += ['djhtml.middleware.DjHTMLMiddleware']
+
+(Caution: when used in production, it is advised to use some kind of
+[caching](https://docs.djangoproject.com/en/stable/topics/cache/).)
 
 
 ## `fmt:off` and `fmt:on`
@@ -98,7 +127,7 @@ You can exclude specific lines from being processed with the
      /     .\
      \_,--._/
     {# fmt:on #}
-"/>
+           "/>
 ```
 
 Contents inside `<pre> ... </pre>`, `<!-- ... --->`, `/* ... */`, and
@@ -124,7 +153,7 @@ The indenter operates in one of three different modes:
 
 ## pre-commit configuration
 
-The best way to use DjHTML is as a [pre-commit](https://pre-commit.com/)
+A great way to use DjHTML is as a [pre-commit](https://pre-commit.com/)
 hook, so all your HTML, CSS and JavaScript files will automatically be
 indented upon every commit.
 
@@ -203,6 +232,7 @@ happy, please do the following:
 
 Your feedback for improving DjHTML is very welcome!
 
+
 ## Development
 
 Use your preferred system for setting up a virtualenv, docker environment,
diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 5368a55..aa70951 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -53,9 +53,6 @@ def main():
 
         # Indent input file
         try:
-            if options.debug:
-                print(Mode(source).debug())
-                sys.exit()
             result = Mode(source).indent(options.tabwidth)
         except Exception:
             _error(
@@ -66,8 +63,7 @@ def main():
             )
             raise
 
-        changed = _verify_changed(source, result)
-        if changed:
+        if changed := _verify_changed(source, result):
             changed_files += 1
         else:
             unchanged_files += 1
@@ -102,6 +98,9 @@ def main():
             f"{problematic_files} template{s} could not be processed due to an error."
         )
 
+    if options.debug:
+        print(Mode(source).debug(), file=sys.stderr)
+
     # Exit with appropriate exit status
     if problematic_files:
         sys.exit(123)
diff --git a/djhtml/lines.py b/djhtml/lines.py
index 6314562..0e67985 100644
--- a/djhtml/lines.py
+++ b/djhtml/lines.py
@@ -2,33 +2,25 @@ class Line:
     """
     A single output line not including the final newline.
 
-    The behavior regarding final newlines has changed between DjHTML
-    v1.4.14 and v1.5.0. It used to always append the final newline,
-    but now this will only happen when the source file already
-    contains a final newline.
-
-    See https://github.com/rtts/djhtml/issues/56 for the discussion
-    that led to this change.
-
     """
 
-    def __init__(self, nr=1):
-        self.nr = nr
-        self.tokens = []
-        self.level = 0
+    def __init__(self, tokens=None, level=0, offset=0, ignore=False):
+        self.tokens = tokens or []
+        self.level = level
+        self.offset = offset
+        self.ignore = ignore
 
     def append(self, token):
         """
         Append tokens to the line.
 
         """
-        token.line_nr = self.nr
         self.tokens.append(token)
 
     @property
     def text(self):
         """
-        The unindented text of this line without leading/trailing spaces.
+        The text of this line without leading/trailing spaces.
 
         """
         return "".join([str(token) for token in self.tokens]).strip()
@@ -39,20 +31,22 @@ def indent(self, tabwidth):
         and optionally offset before calling this method.
 
         """
-        if self.tokens:
-            if self.tokens[0].ignore:
-                return "".join([str(token) for token in self.tokens])
-            elif self.text:
-                offset = self.tokens[0].offset * tabwidth
-                spaces = tabwidth * self.level + offset
-                return " " * spaces + self.text
+        if self.ignore:
+            return "".join([str(token) for token in self.tokens])
+        if self.text:
+            spaces = tabwidth * self.level + self.offset
+            return " " * spaces + self.text
         return ""
 
-    def __repr__(self):
-        return repr(self.tokens)
-
     def __bool__(self):
-        return bool(self.tokens and self.text)
+        return bool(self.tokens)
+
+    def __len__(self):
+        return len(self.text)
 
-    def __next__(self):
-        return Line(nr=self.nr + 1)
+    def __repr__(self):
+        kwargs = ""
+        for attr in ["level", "offset", "ignore"]:
+            if value := getattr(self, attr):
+                kwargs += f", {attr}={value}"
+        return f"Line({repr(self.tokens)}{kwargs})"
diff --git a/djhtml/middleware.py b/djhtml/middleware.py
new file mode 100644
index 0000000..0a04e5d
--- /dev/null
+++ b/djhtml/middleware.py
@@ -0,0 +1,35 @@
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+from djhtml.modes import DjHTML
+
+try:
+    TABWIDTH = settings.TABWIDTH
+except AttributeError:
+    TABWIDTH = 4
+
+
+class DjHTMLMiddleware:
+    """
+    Django middleware class to indent HTML responses.
+
+    """
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        if (
+            "Content-Type" in response
+            and response["Content-Type"] == "text/html; charset=utf-8"
+        ):
+            if "Content-Length" in response:
+                raise ImproperlyConfigured(
+                    "Please load DjHTMLMiddleware _after_ CommonMiddleware (otherwise"
+                    ' the "Content-Length" header will be incorrect)'
+                )
+            response.content = (
+                DjHTML(response.content.decode()).indent(TABWIDTH).encode()
+            )
+        return response
diff --git a/djhtml/modes.py b/djhtml/modes.py
index d032a21..306f294 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -6,7 +6,7 @@
 
 class DjTXT:
     """
-    Mode for indenting text files that contain Django/Jinja template tags.
+    Mode for indenting Django/Jinja template tags.
     Also serves as the base class for the other modes:
 
     - DjHTML
@@ -20,13 +20,19 @@ class DjTXT:
         r"{%[-\+]?.*?[-\+]?%}",
         r"{#.*?#}",
         r"{#",
+        r"{{.*?}}",
     ]
-    OPENING_AND_CLOSING_TAGS = [
+    CLOSING_AND_OPENING_TAGS = [
         "elif",
         "else",
         "empty",
         "plural",
     ]
+    COMMENT_TAGS = [
+        "comment",
+        "verbatim",
+        "raw",
+    ]
     AMBIGUOUS_BLOCK_TAGS = {
         # token_name: (regex_if_block, regex_if_not_block)
         "set": (None, " = "),
@@ -40,6 +46,8 @@ def __init__(self, source="", return_mode=None):
         self.source = source
         self.return_mode = return_mode or self
         self.token_re = compile_re(self.RAW_TOKENS)
+        self.offsets = {"relative": 0, "absolute": 0}
+        self.previous_offsets = []
 
     def indent(self, tabwidth):
         """
@@ -60,61 +68,44 @@ def parse(self):
             first_token = True
             for token in line.tokens:
                 opening_token = None
-
-                # When a dedenting token is found, match it with the
-                # token at the top of the stack.
-                if token.dedents:
-                    try:
-                        if (
-                            stack[-1].kind == token.kind
-                            and stack[-1].is_hard == token.is_hard
-                        ):
+                if stack:
+                    # When a dedenting token is found, match it with
+                    # the token at the top of the stack. To find out
+                    # what each statement does, uncomment it and see
+                    # which unittests fail.
+                    if token.dedents:
+                        if stack[-1].mode is token.mode:
                             opening_token = stack.pop()
-                        elif token.kind == "django":
+                            if stack and opening_token.is_double:
+                                opening_token = stack.pop()
+                        elif token.mode is DjTXT:
                             opening_token = stack.pop()
-                            while opening_token.kind != token.kind:
+                            while opening_token.mode is not DjTXT:
                                 opening_token = stack.pop()
                         elif first_token:
-                            # This closing token could not be matched.
-                            # Instead of erroring out, set the line level
-                            # to what it would have been with a
-                            # regular text token.
-
-                            # If there is any OpenHard token in the
-                            # set and current token is CloseHard then
-                            # let's move back to OpenHard.
-                            if token.is_hard and any(
-                                t.is_hard and t.indents for t in stack
-                            ):
-                                s = stack.pop()
-                                while not s.is_hard or not s.indents:
-                                    s = stack.pop()
+                            line.level = stack[-1].level + 1
+
+                        # Dedent!
+                        if first_token and opening_token:
+                            line.level = opening_token.level
 
+                    else:
+                        # Indent!
+                        if token.is_double and stack[-1].is_double:
+                            opening_token = stack.pop()
+                        if stack and first_token:
                             line.level = stack[-1].level + 1
-                    except IndexError:
-                        line.level = 0
-
-                    # If this dedenting token is the first in line,
-                    # set the line level to the line level of the
-                    # corresponding opening token.
-                    if first_token and opening_token:
-                        line.level = opening_token.level
-
-                # If the first token is not a dedenting token, the
-                # line level will be one higher than that of the token
-                # at the top of the stack.
-                elif first_token:
-                    line.level = stack[-1].level + 1 if stack else 0
-
-                # Push indenting tokens onto the stack. Note that some
-                # tokens can be both indenting and dedenting (e.g.,
-                # ``{% else %}``), hence the if instead of elif.
+
+                if first_token:
+                    line.level = line.level + token.relative
+                    line.offset = token.absolute
+                    line.ignore = token.ignore
+
+                # Push indenting tokens onto the stack.
                 if token.indents:
                     token.level = opening_token.level if opening_token else line.level
                     stack.append(token)
 
-                # Subsequent tokens have no effect on the line level
-                # (but tokens with only spaces don't count).
                 if token.text.strip():
                     first_token = False
 
@@ -137,72 +128,75 @@ def tokenize(self):
             except ValueError:
                 # We've reached the final line!
                 if src:
-                    line.append(mode.create_token(src, ""))
+                    token, _ = mode.create_token(src, "", line)
+                    line.append(token)
                 self.lines.append(line)
                 break
 
             if head:
-                # Create a token from the head. This will always be a
-                # text token (and the next mode will always be the
-                # current mode), but we don't assume that here.
-                line.append(mode.create_token(head, raw_token + tail))
-                mode = next(mode)
+                # Create a token from the head.
+                token, mode = mode.create_token(head, raw_token + tail, line)
+                line.append(token)
 
             if raw_token == "\n":
                 self.lines.append(line)
-                line = next(line)
+                line = Line()
 
             else:
-                # Ask the mode to create a token and to provide a new
-                # mode for the next iteration of the loop.
-                line.append(mode.create_token(raw_token, tail))
-                mode = next(mode)
+                # Create a token from the tail
+                token, mode = mode.create_token(raw_token, tail, line)
+                line.append(token)
 
             # Set the new source to the old tail for the next iteration.
             src = tail
 
-    def create_token(self, raw_token, src):
+    def create_token(self, raw_token, src, line):
         """
-        Given a raw token string, return a single token (and internally
-        set the next mode).
+        Given a raw token string, return a single token and the
+        next mode.
 
         """
-        kind = "django"
-        self.next_mode = self
-        token = Token.Text(raw_token)
+        mode = self
 
-        tag = re.match(self.OPENING_TAG, raw_token)
-        if tag:
+        if tag := re.match(self.OPENING_TAG, raw_token):
             name = tag.group(1)
-            if name in ["comment", "verbatim", "raw"]:
-                token = Token.Open(raw_token, kind)
-                self.next_mode = Comment(
-                    r"{%[-\+]? *end" + name + r"(?: .*?|)%}", self, kind
+            if name in self.COMMENT_TAGS:
+                token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True,), Comment(
+                    r"{%[-\+]? *end" + name + r"(?: .*?|)%}",
+                    mode=DjTXT,
+                    return_mode=self,
                 )
             elif self._has_closing_token(name, raw_token, src):
-                token = Token.Open(raw_token, kind)
-            elif name in self.OPENING_AND_CLOSING_TAGS:
-                token = Token.OpenAndClose(raw_token, kind)
+                self.previous_offsets.append(self.offsets["relative"])
+                token = Token.Open(raw_token, mode=DjTXT, **self.offsets)
+                self.offsets["relative"] = 0
+            elif name in self.CLOSING_AND_OPENING_TAGS:
+                token = Token.CloseAndOpen(raw_token, mode=DjTXT, **self.offsets)
             elif name.startswith("end"):
-                token = Token.Close(raw_token, kind)
-            return token
-
-        if re.match(self.FMT_OFF, raw_token):
-            token = Token.Open(raw_token, kind)
-            self.next_mode = Comment(self.FMT_ON, self, kind)
-            return token
-
-        if raw_token == "{#":
-            token = Token.Open(raw_token, kind)
-            self.next_mode = Comment(r"#\}", self, kind)
-
-        return token
+                token = Token.Close(raw_token, mode=DjTXT, **self.offsets)
+                try:
+                    self.offsets["relative"] = self.previous_offsets.pop()
+                except IndexError:
+                    self.offsets["relative"] = 0
+            else:
+                token = Token.Text(raw_token, mode=DjTXT, **self.offsets)
+        elif re.match(self.FMT_OFF, raw_token):
+            token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
+                self.FMT_ON, mode=DjTXT, return_mode=self
+            )
+        elif raw_token == "{#":
+            token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
+                r"#\}", mode=DjTXT, return_mode=self
+            )
+        else:
+            token = Token.Text(raw_token, mode=self.__class__, **self.offsets)
+
+        return token, mode
 
     def _has_closing_token(self, name, raw_token, src):
         if not re.search(f"{{%[-\\+]? *end{name}(?: .*?|)%}}", src):
             return False
-        regex = self.AMBIGUOUS_BLOCK_TAGS.get(name)
-        if regex:
+        if regex := self.AMBIGUOUS_BLOCK_TAGS.get(name):
             if regex[0]:
                 return re.search(regex[0], raw_token)
             if regex[1]:
@@ -211,12 +205,8 @@ def _has_closing_token(self, name, raw_token, src):
 
     def debug(self):
         self.tokenize()
-        return "\n".join(
-            [" ".join([repr(token) for token in line.tokens]) for line in self.lines]
-        )
-
-    def __next__(self):
-        return self.next_mode
+        self.parse()
+        return "\n".join([repr(line) for line in self.lines])
 
 
 class DjHTML(DjTXT):
@@ -253,35 +243,33 @@ class DjHTML(DjTXT):
         "wbr",
     ]
 
-    def create_token(self, raw_token, src):
-        kind = "html"
-        self.next_mode = self
+    def create_token(self, raw_token, src, line):
+        mode = self
 
         if raw_token == "<":
-            tag = re.match(r"(\w+)[^:]", src)
-            if tag:
-                token = Token.Open(raw_token, kind)
-                self.next_mode = InsideHTMLTag(tag[1], self)
+            if tag := re.match(r"([\w\-\.:]+)", src):
+                token, mode = Token.Open(raw_token, mode=DjHTML), InsideHTMLTag(
+                    tag[1], line, self
+                )
             else:
-                token = Token.Text(raw_token)
-            return token
-
-        if raw_token == "<!--":
-            self.next_mode = Comment("-->", self, kind)
-            return Token.Open(raw_token, kind)
-
-        if re.match("<pre.*?>", raw_token):
-            self.next_mode = Comment("</pre>", self, kind)
-            return Token.Open(raw_token, kind)
-
-        if raw_token.startswith("</"):
-            tagname = re.search(r"\w+", raw_token)
-            if tagname:
+                token = Token.Text(raw_token, mode=DjHTML)
+        elif raw_token == "<!--":
+            token, mode = Token.Open(raw_token, mode=DjHTML, ignore=True), Comment(
+                "-->", mode=DjHTML, return_mode=self
+            )
+        elif re.match("<pre.*?>", raw_token):
+            token, mode = Token.Open(raw_token, mode=DjHTML, ignore=True), Comment(
+                "</pre>", mode=DjHTML, return_mode=self
+            )
+        elif raw_token.startswith("</"):
+            token = Token.Close(raw_token, mode=DjHTML)
+            if tagname := re.search(r"\w+", raw_token):
                 if tagname[0].lower() in self.IGNORE_TAGS:
-                    return Token.Text(raw_token)
-            return Token.Close(raw_token, kind)
+                    token = Token.Text(raw_token, mode=DjHTML)
+        else:
+            token, mode = super().create_token(raw_token, src, line)
 
-        return super().create_token(raw_token, src)
+        return token, mode
 
 
 class DjCSS(DjTXT):
@@ -294,24 +282,44 @@ class DjCSS(DjTXT):
         r"</style>",
         r"[\{\(\)\}]",
         r"/\*",
+        r"[\w-]+: ",
+        r";",
     ]
 
-    def create_token(self, raw_token, src):
-        kind = "css"
-        self.next_mode = self
+    def create_token(self, raw_token, src, line):
+        mode = self
 
         if raw_token in "{(":
-            return Token.Open(raw_token, kind)
-        if raw_token in "})":
-            return Token.Close(raw_token, kind)
-        if raw_token == "/*":
-            self.next_mode = Comment(r"\*/", self, kind)
-            return Token.Open(raw_token, kind)
-        if raw_token == "</style>":
-            self.next_mode = self.return_mode
-            return Token.Close(raw_token, "html")
-
-        return super().create_token(raw_token, src)
+            self.previous_offsets.append(self.offsets["relative"])
+            token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
+            self.offsets["relative"] = 0
+        elif raw_token in "})":
+            token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
+            try:
+                self.offsets["relative"] = self.previous_offsets.pop()
+            except IndexError:
+                self.offsets["relative"] = 0
+        elif raw_token.endswith(": "):
+            token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
+            self.offsets["relative"] = -1
+            self.offsets["absolute"] = len(raw_token)
+        elif raw_token == ";":
+            self.offsets["relative"] = 0
+            self.offsets["absolute"] = 0
+            token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
+        elif raw_token == "/*":
+            token, mode = Token.Open(raw_token, mode=DjCSS, ignore=True), Comment(
+                r"\*/", mode=DjCSS, return_mode=self
+            )
+        elif raw_token == "</style>":
+            token, mode = (
+                Token.Close(raw_token, mode=self.return_mode.__class__),
+                self.return_mode,
+            )
+        else:
+            token, mode = super().create_token(raw_token, src, line)
+
+        return token, mode
 
 
 class DjJS(DjTXT):
@@ -331,51 +339,40 @@ class DjJS(DjTXT):
         r"/\*",
     ]
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.hard_indents = 0
-        self.opened_brackets = 0
-
-    def create_token(self, raw_token, src):
-        kind = "javascript"
-        self.next_mode = self
-
-        if raw_token.strip() == "switch":
-            self.hard_indents = self.hard_indents + 1
-
-        if self.hard_indents:
-            if raw_token == "{":
-                self.opened_brackets += 1
-                if self.opened_brackets == self.hard_indents:
-                    return Token.OpenHard(raw_token, kind)
-            elif raw_token == "}":
-                if self.opened_brackets == self.hard_indents:
-                    self.opened_brackets -= 1
-                    self.hard_indents -= 1
-                    return Token.CloseHard(raw_token, kind)
-                self.opened_brackets -= 1
+    def create_token(self, raw_token, src, line):
+        mode = self
 
         if raw_token in "{[(":
-            return Token.Open(raw_token, kind)
-        if raw_token in ")]}":
-            return Token.Close(raw_token, kind)
-        if raw_token == "`":
-            self.next_mode = Comment("`", self, kind)
-            return Token.Open(raw_token, kind)
-        if raw_token == "/*":
-            self.next_mode = Comment(r"\*/", self, kind)
-            return Token.Open(raw_token, kind)
-        if raw_token.lstrip().startswith("."):
-            return Token.Text(raw_token, offset=1)
-        if raw_token.lstrip().startswith("case "):
-            return Token.OpenAndClose(raw_token, kind)
-        if raw_token.lstrip().startswith("default:"):
-            return Token.OpenAndClose(raw_token, kind)
-        if raw_token == "</script>":
-            self.next_mode = self.return_mode
-            return Token.Close(raw_token, "html")
-
-        return super().create_token(raw_token, src)
+            token = Token.Open(raw_token, mode=DjJS, **self.offsets)
+        elif raw_token in ")]}":
+            token = Token.Close(raw_token, mode=DjJS, **self.offsets)
+        elif raw_token == "`":
+            token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
+                "`", mode=DjJS, return_mode=self
+            )
+        elif raw_token == "/*":
+            token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
+                r"\*/", mode=DjJS, return_mode=self
+            )
+        elif not line and raw_token.lstrip().startswith("."):
+            self.offsets["relative"] = 1
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
+        elif raw_token.lstrip().startswith(("case ", "default:")):
+            token = Token.OpenDouble(raw_token, mode=DjJS)
+            return token, mode
+        elif raw_token == "</script>":
+            token, mode = (
+                Token.Close(raw_token, mode=self.return_mode.__class__),
+                self.return_mode,
+            )
+        else:
+            token, mode = super().create_token(raw_token, src, line)
+
+        # Reset relative offset after creating first token in line.
+        if not line:
+            self.offsets["relative"] = 0
+
+        return token, mode
 
 
 # The following are "special" modes with different constructors.
@@ -387,18 +384,16 @@ class Comment(DjTXT):
 
     """
 
-    def __init__(self, endtag, return_mode, kind):
+    def __init__(self, endtag, *, mode, return_mode):
         self.endtag = endtag
+        self.mode = mode
         self.return_mode = return_mode
-        self.kind = kind
         self.token_re = compile_re([r"\n", endtag])
 
-    def create_token(self, raw_token, src):
-        self.next_mode = self
+    def create_token(self, raw_token, src, line):
         if re.match(self.endtag, raw_token):
-            self.next_mode = self.return_mode
-            return Token.Close(raw_token, self.kind)
-        return Token.Ignore(raw_token)
+            return Token.Close(raw_token, mode=self.mode, ignore=True), self.return_mode
+        return Token.Text(raw_token, mode=Comment, ignore=True), self
 
 
 class InsideHTMLTag(DjTXT):
@@ -407,43 +402,63 @@ class InsideHTMLTag(DjTXT):
 
     """
 
-    RAW_TOKENS = DjTXT.RAW_TOKENS + [r"/?>", r'"']
+    RAW_TOKENS = DjTXT.RAW_TOKENS + [r"/?>", r"[^ ='\">/]+=", r'"']
 
-    def __init__(self, tagname, return_mode):
+    def __init__(self, tagname, line, return_mode):
         self.tagname = tagname
         self.return_mode = return_mode
         self.token_re = compile_re(self.RAW_TOKENS)
         self.inside_attr = False
+        self.offsets = dict(relative=-1, absolute=len(line) + len(tagname) + 2)
+        self.previous_offsets = []
 
-    def create_token(self, raw_token, src):
-        kind = "html"
-        self.next_mode = self
+        # Pff...
+        self.additional_offset = -len(tagname) - 1
+
+    def create_token(self, raw_token, src, line):
+        mode = self
+
+        if line:
+            self.additional_offset += len(raw_token)
+        else:
+            self.additional_offset = 0
+
+        if "text/template" in raw_token:
+            self.tagname = ""
 
         if raw_token == '"':
             if self.inside_attr:
                 self.inside_attr = False
-                return Token.Close(raw_token, kind)
-            self.inside_attr = True
-            return Token.Open(raw_token, kind)
+                self.offsets["absolute"] -= 1
+                token = Token.Text(raw_token, mode=InsideHTMLTag, **self.offsets)
+                self.offsets["absolute"] = self.previous_offset
+            else:
+                self.inside_attr = True
+                self.previous_offset = self.offsets["absolute"]
+                self.offsets["absolute"] += self.additional_offset
+                token = Token.Text(raw_token, mode=InsideHTMLTag, **self.offsets)
         elif not self.inside_attr and raw_token == "/>":
-            self.next_mode = self.return_mode
-            return Token.Close(raw_token, kind)
+            token, mode = Token.Close(raw_token, mode=DjHTML), self.return_mode
         elif not self.inside_attr and raw_token == ">":
             if self.tagname.lower() in DjHTML.IGNORE_TAGS:
-                self.next_mode = self.return_mode
-                return Token.Close(raw_token, kind)
+                token, mode = Token.Close(raw_token, mode=DjHTML), self.return_mode
+            elif self.tagname == "style":
+                token, mode = Token.CloseAndOpen(raw_token, mode=DjHTML), DjCSS(
+                    return_mode=self.return_mode
+                )
+            elif self.tagname == "script":
+                token, mode = Token.CloseAndOpen(raw_token, mode=DjHTML), DjJS(
+                    return_mode=self.return_mode
+                )
             else:
-                if self.tagname == "style":
-                    self.next_mode = DjCSS(return_mode=self.return_mode)
-                elif self.tagname == "script":
-                    self.next_mode = DjJS(return_mode=self.return_mode)
-                else:
-                    self.next_mode = self.return_mode
-                return Token.OpenAndClose(raw_token, kind)
-        elif "text/template" in raw_token:
-            self.tagname = ""
+                token, mode = (
+                    Token.CloseAndOpen(raw_token, mode=DjHTML),
+                    self.return_mode,
+                )
+        else:
+            token, mode = super().create_token(raw_token, src, line)
 
-        return super().create_token(raw_token, src)
+        return token, mode
 
 
 def compile_re(raw_tokens):
diff --git a/djhtml/tokens.py b/djhtml/tokens.py
index 308ef49..cb12aee 100644
--- a/djhtml/tokens.py
+++ b/djhtml/tokens.py
@@ -8,37 +8,41 @@ class _Base:
         indents = False
         dedents = False
         ignore = False
-        is_hard = False
+        is_double = False
 
-        def __init__(self, text, kind="", offset=0):
+        def __init__(
+            self, text, *, mode, level=0, relative=0, absolute=0, ignore=False
+        ):
             self.text = text
-            self.kind = kind
-            self.offset = offset
+            self.mode = mode
+            self.level = level
+            self.relative = relative
+            self.absolute = absolute
+            self.ignore = ignore
 
         def __str__(self):
             return self.text
 
         def __repr__(self):
-            return f"({self.__class__.__name__}:{repr(self.text)})"
+            kwargs = f", mode={self.mode.__name__}"
+            for attr in ["level", "relative", "absolute", "ignore"]:
+                if value := getattr(self, attr):
+                    kwargs += f", {attr}={value}"
+            return f"{self.__class__.__name__}('{self.text}'{kwargs})"
 
     class Text(_Base):
         pass
 
-    class Ignore(_Base):
-        ignore = True
-
     class Open(_Base):
         indents = True
 
-    class OpenHard(Open):
-        is_hard = True
+    class OpenDouble(_Base):
+        indents = True
+        is_double = True
 
     class Close(_Base):
         dedents = True
 
-    class CloseHard(Close):
-        is_hard = True
-
-    class OpenAndClose(_Base):
+    class CloseAndOpen(_Base):
         indents = True
         dedents = True
diff --git a/noxfile.py b/noxfile.py
index 0ff90eb..0942baf 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -1,6 +1,6 @@
 import nox
 
 
-@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"])
+@nox.session(python=["3.8", "3.9", "3.10", "3.11"])
 def tests(session):
     session.run("python", "-m", "unittest", "discover", "-v")
diff --git a/setup.cfg b/setup.cfg
index ca589da..8a2419c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = djhtml
-version = 2.0.0
+version = 3.0.0
 description = Django/Jinja template indenter
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -11,15 +11,13 @@ license = GPLv3+
 classifiers =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.6
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
     License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
     Operating System :: OS Independent
-python_requires = '>=3.6'
+python_requires = '>=3.8'
 
 [options]
 packages = find:
@@ -27,9 +25,6 @@ packages = find:
 [options.extras_require]
 dev =
     nox
-    isort
-    black
-    flake8
     pre-commit
 
 [options.packages.find]
diff --git a/tests/suite/css.html b/tests/suite/css.html
new file mode 100644
index 0000000..17427f2
--- /dev/null
+++ b/tests/suite/css.html
@@ -0,0 +1,53 @@
+<!-- Basic CSS -->
+<style>
+    a {
+        text-decoration: none;
+    }
+</style>
+
+
+<!-- Nested SCSS -->
+<style>
+    a {
+        text-decoration: none;
+        &:hover {
+            text-decoration: underline;
+        }
+    }
+</style>
+
+
+<!-- Multiline comment -->
+<style>
+    /*
+     * Hi mom!
+     *
+     */
+</style>
+
+
+<!-- CSS Function -->
+<style>
+    #gradient {
+        background: linear-gradient(
+                        to top left,
+                        rgba(204,0,0,1),
+                        rgba(204,0,0,0)
+                    );
+    }
+</style>
+
+
+<!-- @font-face typical usage -->
+<style>
+    @font-face {
+        font-family: 'Open Sans';
+        font-style: normal;
+        font-weight: normal;
+        src: url('Open_Sans.woff2') format('woff2'),
+             url('Open_Sans.woff') format('woff'),
+             url('Open_Sans.eot?#iefix') format('embedded-opentype'),
+             url('Open_Sans.svg#OpenSans') format('svg'),
+             url('Open_Sans.ttf') format('truetype');
+    }
+</style>
diff --git a/tests/suite/css.in b/tests/suite/css.in
deleted file mode 100644
index 4ee1693..0000000
--- a/tests/suite/css.in
+++ /dev/null
@@ -1,38 +0,0 @@
-<!-- Basic CSS -->
-<style>
-a {
-text-decoration: none;
-}
-</style>
-
-
-<!-- Nested SCSS -->
-<style>
-a {
-text-decoration: none;
-&:hover {
-text-decoration: underline;
-}
-}
-</style>
-
-
-<!-- Multiline comment -->
-<style>
-    /*
-     * Hi mom!
-     *
-     */
-</style>
-
-
-<!-- CSS Function -->
-<style>
-#gradient {
-background: linear-gradient(
-to top left,
-rgba(204,0,0,1),
-rgba(204,0,0,0)
-);
-}
-</style>
diff --git a/tests/suite/css.out b/tests/suite/css.out
deleted file mode 100644
index 359adf0..0000000
--- a/tests/suite/css.out
+++ /dev/null
@@ -1,38 +0,0 @@
-<!-- Basic CSS -->
-<style>
-    a {
-        text-decoration: none;
-    }
-</style>
-
-
-<!-- Nested SCSS -->
-<style>
-    a {
-        text-decoration: none;
-        &:hover {
-            text-decoration: underline;
-        }
-    }
-</style>
-
-
-<!-- Multiline comment -->
-<style>
-    /*
-     * Hi mom!
-     *
-     */
-</style>
-
-
-<!-- CSS Function -->
-<style>
-    #gradient {
-        background: linear-gradient(
-            to top left,
-            rgba(204,0,0,1),
-            rgba(204,0,0,0)
-        );
-    }
-</style>
diff --git a/tests/suite/django.out b/tests/suite/django.html
similarity index 84%
rename from tests/suite/django.out
rename to tests/suite/django.html
index faf9e9b..80b8fba 100644
--- a/tests/suite/django.out
+++ b/tests/suite/django.html
@@ -1,3 +1,13 @@
+<!-- The example from the README -->
+<div class="
+    {# fmt:off #}
+      ,-._|\
+     /     .\
+     \_,--._/
+    {# fmt:on #}
+           "/>
+
+
 <!-- Basic everyday Django template -->
 {% load static %}
 {% block content %}
@@ -110,3 +120,19 @@ <h1>Welcome, {{ request.user }}</h1>
         {% endwith %}
     {% endslot %}
 {% endcomponent_block %}
+
+
+<!-- Don't indent template variables -->
+{{ '<b>' if condition }}
+some text
+{{ '</b>' if condition }}
+more text
+
+
+<!-- Unless they span multiple lines -->
+{{ '<b>'
+    if condition }}
+    some text
+    {{ '</b>
+if condition }}
+more text
diff --git a/tests/suite/django.in b/tests/suite/django.in
deleted file mode 100644
index 65bd342..0000000
--- a/tests/suite/django.in
+++ /dev/null
@@ -1,112 +0,0 @@
-<!-- Basic everyday Django template -->
-{% load static %}
-{% block content %}
-{% if request.user.is_authenticated %}
-<h1>Welcome, {{ request.user }}</h1>
-{% else %}
-<a href="{% url 'login' %}">Click here to login</a>
-{% endif %}
-{% endblock %}
-
-
-<!-- Django tags within <style> -->
-<style>
-a {
-{% block color %}
-color: red;
-{% endblock %}
-}
-</style>
-
-
-<!-- Django tags within <script> -->
-<script>
-$(function() {
-{% if verbose %}
-console.log("Hi mom!");
-{% endif %}
-});
-</script>
-
-
-<!-- Multiline comment -->
-{% comment %}
-      ,-._|\
-     /     .\
-     \_,--._/
-{% endcomment %}
-
-
-<!-- Verbatim block -->
-{% verbatim %}
-      ,-._|\
-     /     .\
-     \_,--._/
-{% endverbatim %}
-
-
-<!-- Multiline comment with optional comment -->
-{% comment "Yo dawg" %}
-            I heard you like comments
-{% endcomment %}
-
-
-<!-- Various spacings -->
-{%comment    %}
-             %}
-{%    endcomment%}
-{%if%}
-{%block content    %}
-Yuck!
-{%    endblock%}
-{%    endif    %}
-
-
-<!-- Inconsistent nesting -->
-{% block %}
-1
-{% with %}
-2
-{% endblock %}
-1
-{% endwith %}
-
-
-<!-- Tags that both dedent and indent -->
-{% blocktrans count count=queryset.count %}
-There is 1 item.
-{% plural %}
-There are {{ count }} items.
-{% endblocktrans %}
-
-
-<!-- None of this should be indented -->
-{% if %}
-{% elif %}
-{% else %}
-{% endif %}
-{% for %}
-{% empty %}
-{% endfor %}
-
-
-<!-- Template tag directly following tag name -->
-<a{% if %} class=""{% endif %}>
-Click here!
-</a>
-
-
-<!-- Ambiguous video tag from django-embed-video -->
-{% video form.instance.video as my_video %}
-{% video my_video '360 x 200' %}
-{% endvideo %}
-
-
-<!-- Match the tag, the whole tag, and nothing but the tag -->
-{% component_block "instruction" border_top=True header="Login" %}
-{% slot "body" %}
-{% with "login --token="|add:token as content %}
-{% component "copy_field" field_selector="token-field" content=content %}
-{% endwith %}
-{% endslot %}
-{% endcomponent_block %}
diff --git a/tests/suite/html.html b/tests/suite/html.html
new file mode 100644
index 0000000..cf13b31
--- /dev/null
+++ b/tests/suite/html.html
@@ -0,0 +1,330 @@
+<!-- Basic HTML sanity check -->
+<!doctype html>
+<html>
+    <head>
+        <title>Plain</title>
+        <script src="test.js"></script>
+        <link rel="stylesheet" href="test.css">
+    </head>
+    <body>
+        <h1>Title</h1>
+    </body>
+</html>
+
+
+<!-- Multiple closing/opening tags on same line -->
+<div>
+    text
+</div><div>
+    text
+</div>
+
+
+<!-- Some crazy edgecases that even the mighty web-mode fails to indent correctly -->
+<div>
+    <div>
+        <div>
+            text
+        </div></div></div>
+
+<div><div><div>
+    text
+</div>
+</div>
+</div>
+
+<div0><div1>
+    text2</div1><div1><div2>
+        text3
+    </div2><div2>
+        text3
+    </div2>
+    </div1>
+</div0>
+
+<div0>
+    <div1>
+        <div2>
+            text3
+        </div2></div1></div0><div0>
+            text1
+        </div0>
+
+
+<!-- Ignore self-closing tags -->
+<div>
+    <div/>
+    text
+</div>
+
+
+<!-- Ignore void tags -->
+<div>
+    <img>
+    <input>
+    <input />
+    <input></input>
+    <!doctype>
+</div>
+
+<!-- Don't parse style/script as html -->
+<script>
+    <ul>
+    <li>
+</script>
+
+<style>
+    <style>
+</style>
+
+
+<!-- Except for JS templates... -->
+<script id="id" type="text/template" class="class">
+    <ul>
+        <li/>
+    </ul>
+</script>
+
+
+<!-- Ignore multiline comments -->
+<!--
+ 1
+  2
+   3
+-->
+
+
+<!-- Ignore <pre> elements -->
+<pre>
+   <dl>
+  <dt>
+ <dd>
+</pre>
+
+
+<!-- Ignore both begin and end tags -->
+    <!--
+ -->
+            <pre>
+    </pre>
+       {#
+                 #}
+
+
+<!-- Real life example kindly provided by @jacklinke -->
+<div class="row">
+    <div class="input-group date" data-provide="datepicker" data-date-format="yyyy/mm/dd">
+        <div class="btn">
+        </div></div>
+</div><div>
+    <div class="custom-control custom-switch">
+        <input type="checkbox" name="filter-changes" id="filter-changes" class="custom-control-input fs-nano" value="0" {% if request.session.manage_filter_changes %}{% if request.session.manage_filter_changes == True %}checked{% endif %}{% endif %}>
+    </div>
+    <form class="form-inline">
+        <div class="custom-control custom-switch">
+            <input type="checkbox" name="filter-status-new" id="filter-status-new" class="custom-control-input fs-nano" value="0" {% if request.session.manage_filter_status_new %}{% if request.session.manage_filter_status_new == True %}checked{% endif %}{% endif %}>
+        </div>
+    </form>
+</div>
+
+
+<!-- SVG with multi-line attribute values -->
+<svg width="325" height="325" xmlns="http://www.w3.org/2000/svg">
+    <path
+          d="M 80 80
+             A 45 45, 0, 0, 0, 125 125
+             L 125 80 Z"
+          fill="green"
+    />
+    <path
+          d="
+             M 80 80
+             A 45 45, 0, 0, 0, 125 125
+             L 125 80 Z
+            "
+          fill="green"
+    />
+    <path d="M 80 80
+             A 45 45, 0, 0, 0, 125 125
+             L 125 80 Z"
+          fill="green"
+    />
+</svg>
+
+
+<!-- Greater than character inside tag -->
+<div {% if foo > bar %}class="baz"{% endif %}>
+    quux
+</div>
+
+
+<!-- Block tags inside HTML elements -->
+<div
+     {% block class %}
+         class="foo"
+     {% endblock %}
+>
+    text
+</div>
+
+<div
+     {% block classes %}
+         class="
+                foo
+                bar
+               "
+     {% endblock %}
+>
+    text
+</div>
+
+<div id="1" class="{% block classes %}
+                       foo
+                       bar
+                       {% if %}
+                           baz
+                       {% elif %}
+                           quux
+                       {% endif %}
+                   {% endblock %}
+                  "
+     hidden
+>
+    text
+</div>
+
+
+
+<!-- Non-block tags inside HTML elements -->
+<a href="{% url %}"
+   class="button"
+>
+    Click me!
+</a>
+
+
+<!-- Spurious < and > characters -->
+<div>
+    >
+</div>
+<div>
+    <
+</div>
+
+
+<!-- Arrow inside attribute -->
+<div>
+    <button action="a->b">
+        hi
+    </button>
+</div>
+
+
+<!-- Multi-line elements -->
+<a id="1" href=""
+   class="button">
+    text
+</a>
+<a
+   id="2" href=""
+   class="button">
+    text
+</a>
+<a id="3" href=""
+   class="button"
+>
+    text
+</a>
+<a
+   id="4" href=""
+   class="button"
+>
+    text
+</a>
+
+
+<!-- The example from issue #50
+<img src="/some/long/path"
+     id="1">
+<blockquote class="dishy"
+            id="2" />
+<a href="/some/long/path"
+   id="3"></a>
+
+
+<!-- Multi-line attribute values -->
+<div id="
+         1">
+    <div id="2
+            ">
+        <div id="
+                 3
+                ">
+            <div id="
+                     4"
+            >
+                <div id="5
+                        "
+                >
+                    <div id="
+                             6
+                            "
+                    >
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+
+<!-- Weird but legal attribute names -->
+<div !click="
+            "
+     @click="
+            "
+     #click="
+            "
+     $click="
+            "
+     %click="
+            "
+     ^click="
+            "
+     &click="
+            "
+     *click="
+            "
+     <click="
+            "
+     data-data="
+               "
+>
+    text
+</div>
+
+<!-- Weird but legal element names -->
+<my-component
+              hidden/>
+<my.component
+              hidden/>
+<my:component
+              hidden/>
+
+
+<!-- Multi-line attribute value with additional elements/attributes -->
+<div><blockquote id="1" id="2"
+                 id="3" cite="
+                              Wikipedia
+                             "
+                 id="4"/></div>
+
+
+<!-- The example from the README -->
+<blockquote cite="John Lennon"
+            style="color: dimgray;
+                   font-style: italic;
+                   border-left: 5px solid gray;
+                  ">
+    It's weird not to be weird.
+</blockquote>
diff --git a/tests/suite/html.in b/tests/suite/html.in
deleted file mode 100644
index 2188436..0000000
--- a/tests/suite/html.in
+++ /dev/null
@@ -1,173 +0,0 @@
-<!-- Basic HTML sanity check -->
-<!doctype html>
-<html>
-<head>
-<title>Plain</title>
-<script src="test.js"></script>
-<link rel="stylesheet" href="test.css">
-</head>
-<body>
-<h1>Title</h1>
-</body>
-</html>
-
-
-<!-- Multiple closing/opening tags on same line -->
-<div>
-    text
-</div><div>
-    text
-</div>
-
-
-<!-- Some crazy edgecases that even the mighty web-mode fails to indent correctly -->
-
-<div>
-<div>
-<div>
-text
-</div></div></div>
-
-<div><div><div>
-text
-</div>
-</div>
-</div>
-
-<div0><div1>
-text2</div1><div1><div2>
-text3
-</div2><div2>
-text3
-</div2>
-</div1>
-</div0>
-
-<div0>
-<div1>
-<div2>
-text3
-</div2></div1></div0><div0>
-text1
-</div0>
-
-
-<!-- Tag continuation -->
-<div
-id="id"
-class="class">
-text
-</div>
-
-<div
-id="id"
-class="class"
->
-text
-</div>
-
-
-<!-- Ignore self-closing tags -->
-<div>
-<div/>
-text
-</div>
-
-
-<!-- Ignore void tags -->
-<img>
-<input>
-<input />
-<input></input>
-<!doctype>
-
-
-<!-- Don't parse style/script as html -->
-<script>
-<ul>
-<li>
-</script>
-
-<style>
-<style>
-</style>
-
-
-<!-- Except for JS templates... -->
-<script id="id" type="text/template" class="class">
-<ul>
-<li/>
-</ul>
-</script>
-
-
-<!-- Ignore multiline comments -->
-<!--
- 1
-  2
-   3
--->
-
-
-<!-- Ignore <pre> elements -->
-<pre>
-   <dl>
-  <dt>
- <dd>
-</pre>
-
-
-<!-- Real life example kindly provided by @jacklinke -->
-<div class="row">
-<div class="input-group date" data-provide="datepicker" data-date-format="yyyy/mm/dd">
-<div class="btn">
-</div></div>
-</div><div>
-<div class="custom-control custom-switch">
-<input type="checkbox" name="filter-changes" id="filter-changes" class="custom-control-input fs-nano" value="0" {% if request.session.manage_filter_changes %}{% if request.session.manage_filter_changes == True %}checked{% endif %}{% endif %}>
-</div>
-<form class="form-inline">
-<div class="custom-control custom-switch">
-<input type="checkbox" name="filter-status-new" id="filter-status-new" class="custom-control-input fs-nano" value="0" {% if request.session.manage_filter_status_new %}{% if request.session.manage_filter_status_new == True %}checked{% endif %}{% endif %}>
-</div>
-</form>
-</div>
-
-
-<!-- Multiline elements -->
-<img src="/some/long/path"
-id="1">
-<blockquote class="dishy"
-id="2" />
-<a href="/some/long/path"
-id="3"></a>
-
-<!-- Multiline attribute values -->
-<svg width="325" height="325" xmlns="http://www.w3.org/2000/svg">
-<path
-d="M 80 80
-A 45 45, 0, 0, 0, 125 125
-L 125 80 Z"
-fill="green"
-/>
-<path
-d="
-M 80 80
-A 45 45, 0, 0, 0, 125 125
-L 125 80 Z
-"
-fill="green"
-/>
-<path d="M 80 80
-A 45 45, 0, 0, 0, 125 125
-L 125 80 Z"
-fill="green"
-/>
-</svg>
-
-
-<!-- Remove trailing spaces -->
-Invisible spaces here:   
-
-
-<!-- Do not append final newline -->
\ No newline at end of file
diff --git a/tests/suite/html.out b/tests/suite/html.out
deleted file mode 100644
index 976c393..0000000
--- a/tests/suite/html.out
+++ /dev/null
@@ -1,173 +0,0 @@
-<!-- Basic HTML sanity check -->
-<!doctype html>
-<html>
-    <head>
-        <title>Plain</title>
-        <script src="test.js"></script>
-        <link rel="stylesheet" href="test.css">
-    </head>
-    <body>
-        <h1>Title</h1>
-    </body>
-</html>
-
-
-<!-- Multiple closing/opening tags on same line -->
-<div>
-    text
-</div><div>
-    text
-</div>
-
-
-<!-- Some crazy edgecases that even the mighty web-mode fails to indent correctly -->
-
-<div>
-    <div>
-        <div>
-            text
-        </div></div></div>
-
-<div><div><div>
-    text
-</div>
-</div>
-</div>
-
-<div0><div1>
-    text2</div1><div1><div2>
-        text3
-    </div2><div2>
-        text3
-    </div2>
-    </div1>
-</div0>
-
-<div0>
-    <div1>
-        <div2>
-            text3
-        </div2></div1></div0><div0>
-            text1
-        </div0>
-
-
-<!-- Tag continuation -->
-<div
-    id="id"
-    class="class">
-    text
-</div>
-
-<div
-    id="id"
-    class="class"
->
-    text
-</div>
-
-
-<!-- Ignore self-closing tags -->
-<div>
-    <div/>
-    text
-</div>
-
-
-<!-- Ignore void tags -->
-<img>
-<input>
-<input />
-<input></input>
-<!doctype>
-
-
-<!-- Don't parse style/script as html -->
-<script>
-    <ul>
-    <li>
-</script>
-
-<style>
-    <style>
-</style>
-
-
-<!-- Except for JS templates... -->
-<script id="id" type="text/template" class="class">
-    <ul>
-        <li/>
-    </ul>
-</script>
-
-
-<!-- Ignore multiline comments -->
-<!--
- 1
-  2
-   3
--->
-
-
-<!-- Ignore <pre> elements -->
-<pre>
-   <dl>
-  <dt>
- <dd>
-</pre>
-
-
-<!-- Real life example kindly provided by @jacklinke -->
-<div class="row">
-    <div class="input-group date" data-provide="datepicker" data-date-format="yyyy/mm/dd">
-        <div class="btn">
-        </div></div>
-</div><div>
-    <div class="custom-control custom-switch">
-        <input type="checkbox" name="filter-changes" id="filter-changes" class="custom-control-input fs-nano" value="0" {% if request.session.manage_filter_changes %}{% if request.session.manage_filter_changes == True %}checked{% endif %}{% endif %}>
-    </div>
-    <form class="form-inline">
-        <div class="custom-control custom-switch">
-            <input type="checkbox" name="filter-status-new" id="filter-status-new" class="custom-control-input fs-nano" value="0" {% if request.session.manage_filter_status_new %}{% if request.session.manage_filter_status_new == True %}checked{% endif %}{% endif %}>
-        </div>
-    </form>
-</div>
-
-
-<!-- Multiline elements -->
-<img src="/some/long/path"
-    id="1">
-<blockquote class="dishy"
-    id="2" />
-<a href="/some/long/path"
-    id="3"></a>
-
-<!-- Multiline attribute values -->
-<svg width="325" height="325" xmlns="http://www.w3.org/2000/svg">
-    <path
-        d="M 80 80
-            A 45 45, 0, 0, 0, 125 125
-            L 125 80 Z"
-        fill="green"
-    />
-    <path
-        d="
-            M 80 80
-            A 45 45, 0, 0, 0, 125 125
-            L 125 80 Z
-        "
-        fill="green"
-    />
-    <path d="M 80 80
-        A 45 45, 0, 0, 0, 125 125
-        L 125 80 Z"
-        fill="green"
-    />
-</svg>
-
-
-<!-- Remove trailing spaces -->
-Invisible spaces here:
-
-
-<!-- Do not append final newline -->
\ No newline at end of file
diff --git a/tests/suite/html_arrow_attr.in b/tests/suite/html_arrow_attr.in
deleted file mode 100644
index 2b5d063..0000000
--- a/tests/suite/html_arrow_attr.in
+++ /dev/null
@@ -1,5 +0,0 @@
-<div>
-  <button action="a->b">
-  hi
-  </button>
-</div>
diff --git a/tests/suite/html_arrow_attr.out b/tests/suite/html_arrow_attr.out
deleted file mode 100644
index 3e9880a..0000000
--- a/tests/suite/html_arrow_attr.out
+++ /dev/null
@@ -1,5 +0,0 @@
-<div>
-    <button action="a->b">
-        hi
-    </button>
-</div>
diff --git a/tests/suite/html_tag.in b/tests/suite/html_tag.in
deleted file mode 100644
index c5734c6..0000000
--- a/tests/suite/html_tag.in
+++ /dev/null
@@ -1,21 +0,0 @@
-<!-- Greater than character inside tag -->
-<div {% if foo > bar %}class="baz"{% endif %}>
-quux
-</div>
-
-<!-- Block tags inside multiline HTML tags -->
-<div
-{% block class %}
-class="foo"
-{% endblock %}
->
-text
-</div>
-
-<!-- Spurious < and > characters -->
-<div>
->
-</div>
-<div>
-<
-</div>
diff --git a/tests/suite/html_tag.out b/tests/suite/html_tag.out
deleted file mode 100644
index 9ba31a1..0000000
--- a/tests/suite/html_tag.out
+++ /dev/null
@@ -1,21 +0,0 @@
-<!-- Greater than character inside tag -->
-<div {% if foo > bar %}class="baz"{% endif %}>
-    quux
-</div>
-
-<!-- Block tags inside multiline HTML tags -->
-<div
-    {% block class %}
-        class="foo"
-    {% endblock %}
->
-    text
-</div>
-
-<!-- Spurious < and > characters -->
-<div>
-    >
-</div>
-<div>
-    <
-</div>
diff --git a/tests/suite/jinja.out b/tests/suite/jinja.html
similarity index 84%
rename from tests/suite/jinja.out
rename to tests/suite/jinja.html
index 7664412..b1ed319 100644
--- a/tests/suite/jinja.out
+++ b/tests/suite/jinja.html
@@ -45,3 +45,10 @@
     {% endset %}
     Contents of block is block
 </div>
+
+
+<!-- Don't support multiline if statements (for now) -->
+{% if ( condition1 and
+condition2 ) %}
+text
+{% endif %}
diff --git a/tests/suite/jinja.in b/tests/suite/jinja.in
deleted file mode 100644
index 3cb03fb..0000000
--- a/tests/suite/jinja.in
+++ /dev/null
@@ -1,47 +0,0 @@
-<!-- Multiline comment -->
-{#
-      ,-._|\
-     /     .\
-     \_,--._/
-#}
-
-
-<!-- Raw block -->
-{% raw %}
-      ,-._|\
-     /     .\
-     \_,--._/
-{% endraw %}
-
-
-<!-- Whitespace control -->
-<div>
-{%+ if something %}
-yay
-{% endif %}
-</div>
-
-<div>
-{% if something +%}
-yay
-{% endif %}
-</div>
-
-{% for item in seq -%}
-    {{ item }}
-{%- endfor %}
-
-{%- macro render_currency(currency, amount) -%}
-{{ amount }}
-{%- endmacro -%}
-
-
-<!-- Set assignment -->
-<div>
-{% set "five" = 5 %}
-Five is {{ five }}
-{% set blockdata %}
-Contents
-{% endset %}
-Contents of block is block
-</div>
diff --git a/tests/suite/js.out b/tests/suite/js.html
similarity index 84%
rename from tests/suite/js.out
rename to tests/suite/js.html
index eb4a5bf..881dc44 100644
--- a/tests/suite/js.out
+++ b/tests/suite/js.html
@@ -229,3 +229,42 @@
         'case ',
     ];
 </script>
+
+
+<!-- Another problematic switch statement -->
+<script>
+    switch (foo) {
+        case bar:
+            if (baz) {
+                switch (foo) {
+                    case bar:
+                        if (baz) {
+                            xizzy();
+                        }
+                }
+            }
+    }
+</script>
+
+<!-- More function chaining -->
+<script>
+    window.fetch('/test.html')
+        .then((html) => {
+            document.body.innerHTML = html;
+        });
+
+    foo
+        .bar
+        .baz
+        .then(() => {
+            foo
+                .bar
+                .baz
+                .then(() => {
+                    foo
+                        .bar
+                        .baz
+                });
+        });
+    }
+</script>
diff --git a/tests/suite/js.in b/tests/suite/js.in
deleted file mode 100644
index d17d057..0000000
--- a/tests/suite/js.in
+++ /dev/null
@@ -1,231 +0,0 @@
-<!-- Basic Javascript -->
-<script>
-function f() {
-return 42;
-}
-</script>
-
-
-<!-- Different brace types -->
-<script>
-array = [
-1,
-2,
-3,
-]
-
-dictionary = {
-value1: 1,
-value2: 2,
-value3: 3,
-}
-
-function_call(
-arg1,
-arg2,
-arg3,
-)
-</script>
-
-
-<!-- Function chaining -->
-<script>
-one()
-.two()
-.three();
-four();
-</script>
-
-<script>
-function f() {
-return 42;
-}
-</script>
-
-
-<!-- Double-nested braces -->
-<script>
-$(function() {
-this.addEventListener('change', function () {
-doStuff();
-});
-});
-</script>
-
-
-<!-- Multiline comment -->
-<script>
-    /*
-     * Hi mom!
-     *
-     */
-</script>
-
-
-<!-- Nasty characters embedded in strings -->
-<script>
-console.log("$(function() {");
-console.log('$(function() {');
-console.log(`$(function() {`);
-console.log(`
-function f() {
-return 42;
-}
-`);
-</script>
-
-
-<!-- Real-life example kindly provided by @jacklinke -->
-<script>
-    const format_duration_value = (a) => {
-        let secs = a * 3600;  // get total amount of seconds from your input
-        const days = Math.floor(secs / (3600 * 24));
-        return (`
-            <div class="form-inline text-nowrap text-center">
-                ${days.toString().padStart(2,'0')}
-                <small>m</small>
-            </div>
-        `);
-    };
-</script>
-
-
-<!-- More Javascript shenanigans -->
-<script>
-$(function1()
-{
-var foo = (function2(){
-doStuff();
-})();
-});
-// Jeez
-</script>
-
-
-<!-- If elif else -->
-<script>
-let time = 10;
-if (time < 10) {
-console.log("Good morning");
-} else if (time < 20) {
-console.log("Good day");
-} else {
-console.log("Good evening");
-}
-</script>
-
-
-<!-- Switch case indentation -->
-<script>
-const mode = 'approved'
-switch (mode) {
-case 'approved':
-console.log('Approved');
-break;
-case 'rejected':
-console.log('Rejected');
-break;
-default:
-console.log('Default');
-}
-
-switch (mode) {
-case 'approved':
-console.log('Approved');
-break;
-default:
-console.log('Default');
-case 'rejected':
-console.log('Rejected');
-break;
-}
-
-switch (mode) {
-case 'approved':
-case 'rejected':
-case 'invalid':
-console.log('Not default');
-break;
-default:
-console.log('Default');
-}
-
-switch (mode) {
-case 'approved': {
-console.log('Approved');
-break;
-}
-case 'rejected': {
-console.log('Rejected');
-break;
-}
-default: {
-console.log('Default');
-}
-}
-</script>
-
-
-<!-- Switch complex example -->
-<script>
-var year = 2016;
-var month = 2;
-var dayCount;
-switch (month) {
-case 1:
-function switchFoo () {
-console.log("Foo");
-}
-switchFoo();
-case 12:
-dayCount = 31;
-break;
-case 4:
-switch(boo){
-case 1:
-case 2:
-}
-case 11:
-dayCount = 30;
-break;
-case 2:
-if (((year % 4 == 0) && !(year % 100 == 0)) || (year % 400 == 0)) {
-dayCount = 29;
-} else {
-dayCount = 28;
-}
-break;
-default:
-dayCount = -1; // invalid month
-}
-console.log(dayCount);
-</script>
-
-
-<!-- Nested switch statements -->
-<script>
-switch (a) {
-case b:
-switch (c) {
-case d:
-switch (e) {
-default:
-}
-}
-}
-</script>
-
-
-<!-- Things that look like switch, but aren't -->
-<script>
-function switchBoo () {
-console.log("Boo");
-}
-switchBoo();
-
-let arr = [
-'switch',
-'default:',
-'case ',
-];
-</script>
diff --git a/tests/suite/paradoxes.out b/tests/suite/paradoxes.html
similarity index 90%
rename from tests/suite/paradoxes.out
rename to tests/suite/paradoxes.html
index da8628a..c42b088 100644
--- a/tests/suite/paradoxes.out
+++ b/tests/suite/paradoxes.html
@@ -62,13 +62,3 @@
     console.log("Time to ditch jQuery!")
     });
 </script>
-
-
-<!-- Manually disable formatting -->
-<body>
-    {# fmt:off #}
-     ,-._|\
-    /     .\
-    \_,--._/
-    {# fmt:on #}
-</body>
diff --git a/tests/suite/paradoxes.in b/tests/suite/paradoxes.in
deleted file mode 100644
index 01cde80..0000000
--- a/tests/suite/paradoxes.in
+++ /dev/null
@@ -1,74 +0,0 @@
-<!-- Common pattern in django templates -->
-{% if %}
-<a>
-{% else %}
-<a>
-{% endif %}
-.
-{% if %}
-</a>
-{% endif %}
-
-
-<!-- The literal example from issue #17 -->
-{% if some_condition %}
-    <a href="{% url some_url">
-{% elif some_other_condition %}
-    <a href="{% url some_other_url">
-{% endif %}
-    [a bunch of details go here]
-{% if end_tag_needed %}</a>{% endif %}
-
-
-<!-- Deep nestings should be instantly dedented -->
-{% block %}
-<a>
-<a>
-<a>
-<a>
-<a>
-{% endblock %}
-{% for x in [1,2,3,4,5] %}
-</a>
-{% endfor %}
-
-
-<!-- The most complicated example I can think of -->
-<body>
-{% if %}
-<div>
-{% else %}
-<span>{% if %}<blink>
-{% block %}
-</blink>
-{% endblock %}{% endif %}
-{% endif %}
-.
-{% if %}
-</span>
-{% else %}
-</div>
-{% endif %}
-</body>
-
-
-<!-- Inside <script> tag -->
-<script>
-{% if jquery %}
-$(function() {
-{% else %}
-document.addEventListener("DOMContentLoaded", function(event) {
-{% endif %}
-console.log("Time to ditch jQuery!")
-});
-</script>
-
-
-<!-- Manually disable formatting -->
-<body>
-    {# fmt:off #}
-     ,-._|\
-    /     .\
-    \_,--._/
-    {# fmt:on #}
-</body>
diff --git a/tests/test_suite.py b/tests/test_suite.py
index 2dc5da0..f56054e 100644
--- a/tests/test_suite.py
+++ b/tests/test_suite.py
@@ -8,43 +8,6 @@ class TestSuite(unittest.TestCase):
     maxDiff = None
     DIR = os.path.join(os.path.dirname(__file__), "suite")
 
-    def _test_file(self, basename):
-        with open(os.path.join(self.DIR, f"{basename}.in"), "r") as f:
-            actual_input = f.read()
-
-        try:
-            with open(os.path.join(self.DIR, f"{basename}.out"), "r") as f:
-                expected_output = f.read()
-        except FileNotFoundError:
-            expected_output = None
-
-        try:
-            with open(os.path.join(self.DIR, f"{basename}.err"), "r") as f:
-                expected_error = f.read().strip()
-        except FileNotFoundError:
-            expected_error = None
-
-        if expected_output is None and expected_error is None:
-            self.fail(f"{basename} has no expected error or output.")
-
-        if expected_output is not None and expected_error is not None:
-            self.fail(f"{basename} has both expected error and output.")
-
-        try:
-            actual_output = DjHTML(actual_input).indent(4)
-            actual_error = None
-        except Exception as err:
-            actual_error = str(err).strip()
-            actual_output = None
-
-        if actual_error is not None and expected_error is None:
-            self.fail(f"Unexpected error: {actual_error}")
-        if actual_error is None and expected_error is not None:
-            self.fail(f"Unexpected success, expected error: {expected_error}")
-
-        self.assertEqual(actual_output, expected_output)
-        self.assertEqual(actual_error, expected_error)
-
     def test_available_files(self):
         """
         Loop over all the files in the suite directory and compare the
@@ -52,7 +15,11 @@ def test_available_files(self):
 
         """
         for filename in os.listdir(self.DIR):
-            if filename.endswith(".in"):
-                basename, _ = os.path.splitext(filename)
-                with self.subTest(basename):
-                    self._test_file(basename)
+            with self.subTest(filename):
+                self._test_file(filename)
+
+    def _test_file(self, filename):
+        with open(os.path.join(self.DIR, filename)) as f:
+            expected_output = f.read()
+        actual_output = DjHTML(expected_output).indent(4)
+        self.assertEqual(expected_output, actual_output)

From 9ebcc424573346ca55ebf6c303fe8b501f131177 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Mon, 13 Feb 2023 21:52:17 +0100
Subject: [PATCH 02/45] Introduce CloseDouble token to fix problem with missing
 `;` in CSS

---
 djhtml/modes.py            | 30 ++++++++++++++++++++++--------
 djhtml/tokens.py           |  4 ++++
 tests/suite/css.html       | 24 ++++++++++++++++++++++++
 tests/suite/js.html        | 19 ++++++++++++++++++-
 tests/suite/paradoxes.html | 19 -------------------
 5 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 306f294..72f841d 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -64,6 +64,15 @@ def parse(self):
 
         """
         stack = []
+
+        def mode_in_stack(mode):
+            if stack[-1].mode is DjTXT:
+                return False
+            for token in stack:
+                if token.mode is mode:
+                    return True
+            return False
+
         for line in self.lines:
             first_token = True
             for token in line.tokens:
@@ -76,11 +85,11 @@ def parse(self):
                     if token.dedents:
                         if stack[-1].mode is token.mode:
                             opening_token = stack.pop()
-                            if stack and opening_token.is_double:
+                            if stack and (opening_token.is_double or token.is_double):
                                 opening_token = stack.pop()
-                        elif token.mode is DjTXT:
+                        elif mode_in_stack(token.mode):
                             opening_token = stack.pop()
-                            while opening_token.mode is not DjTXT:
+                            while opening_token.mode is not token.mode:
                                 opening_token = stack.pop()
                         elif first_token:
                             line.level = stack[-1].level + 1
@@ -294,12 +303,17 @@ def create_token(self, raw_token, src, line):
             token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
             self.offsets["relative"] = 0
         elif raw_token in "})":
-            token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
-            try:
-                self.offsets["relative"] = self.previous_offsets.pop()
-            except IndexError:
+            if raw_token == "}" and self.offsets["absolute"]:
+                self.offsets["absolute"] = 0
                 self.offsets["relative"] = 0
-        elif raw_token.endswith(": "):
+                token = Token.CloseDouble(raw_token, mode=DjCSS, **self.offsets)
+            else:
+                token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
+                try:
+                    self.offsets["relative"] = self.previous_offsets.pop()
+                except IndexError:
+                    self.offsets["relative"] = 0
+        elif not len(line) and raw_token.endswith(": "):
             token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
             self.offsets["relative"] = -1
             self.offsets["absolute"] = len(raw_token)
diff --git a/djhtml/tokens.py b/djhtml/tokens.py
index cb12aee..317feee 100644
--- a/djhtml/tokens.py
+++ b/djhtml/tokens.py
@@ -43,6 +43,10 @@ class OpenDouble(_Base):
     class Close(_Base):
         dedents = True
 
+    class CloseDouble(_Base):
+        dedents = True
+        is_double = True
+
     class CloseAndOpen(_Base):
         indents = True
         dedents = True
diff --git a/tests/suite/css.html b/tests/suite/css.html
index 17427f2..0e90975 100644
--- a/tests/suite/css.html
+++ b/tests/suite/css.html
@@ -51,3 +51,27 @@
              url('Open_Sans.ttf') format('truetype');
     }
 </style>
+
+
+<!-- @media -->
+<style>
+    @media(max-width: 800px) {
+        body {
+            margin-top: 1em;
+            margin-bottom: 1em;
+        }
+    }
+</style>
+
+
+<!-- Missing semicolons -->
+<style>
+    p { align: left }
+    h1 {
+        font-weight: bold;
+        font-size: huge
+    }
+    footnote {
+        dontread: true !important
+    }
+</style>
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 881dc44..053f135 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -266,5 +266,22 @@
                         .baz
                 });
         });
-    }
 </script>
+
+
+<!-- Not enough closing tokens -->
+<head>
+    <script>
+        function foo() {
+    </script>
+</head>
+
+<!-- Too much closing tokens -->
+<head>
+    <script>
+        function foo() {
+        }
+        }
+        }
+    </script>
+</head>
diff --git a/tests/suite/paradoxes.html b/tests/suite/paradoxes.html
index c42b088..653231e 100644
--- a/tests/suite/paradoxes.html
+++ b/tests/suite/paradoxes.html
@@ -33,25 +33,6 @@
 {% endfor %}
 
 
-<!-- The most complicated example I can think of -->
-<body>
-    {% if %}
-        <div>
-    {% else %}
-        <span>{% if %}<blink>
-            {% block %}
-                </blink>
-            {% endblock %}{% endif %}
-    {% endif %}
-    .
-    {% if %}
-        </span>
-    {% else %}
-        </div>
-    {% endif %}
-</body>
-
-
 <!-- Inside <script> tag -->
 <script>
     {% if jquery %}

From 7b8cc45806bb5ec5d09fc8eb255763fe00e7a7dc Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 14 Feb 2023 22:26:09 +0100
Subject: [PATCH 03/45] Account for nested SCSS with missing semicolons in
 various indentation styles

---
 djhtml/modes.py      | 10 +++++++++-
 tests/suite/css.html | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 72f841d..7cdc089 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -295,6 +295,10 @@ class DjCSS(DjTXT):
         r";",
     ]
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.inside_block = False
+
     def create_token(self, raw_token, src, line):
         mode = self
 
@@ -302,6 +306,8 @@ def create_token(self, raw_token, src, line):
             self.previous_offsets.append(self.offsets["relative"])
             token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
             self.offsets["relative"] = 0
+            if raw_token == "{":
+                self.inside_block = True
         elif raw_token in "})":
             if raw_token == "}" and self.offsets["absolute"]:
                 self.offsets["absolute"] = 0
@@ -313,7 +319,9 @@ def create_token(self, raw_token, src, line):
                     self.offsets["relative"] = self.previous_offsets.pop()
                 except IndexError:
                     self.offsets["relative"] = 0
-        elif not len(line) and raw_token.endswith(": "):
+            if raw_token == "}":
+                self.inside_block = False
+        elif self.inside_block and raw_token.endswith(": "):
             token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
             self.offsets["relative"] = -1
             self.offsets["absolute"] = len(raw_token)
diff --git a/tests/suite/css.html b/tests/suite/css.html
index 0e90975..6c71e42 100644
--- a/tests/suite/css.html
+++ b/tests/suite/css.html
@@ -75,3 +75,38 @@
         dontread: true !important
     }
 </style>
+
+
+<!-- Nested SCSS with missing semicolons and different indentation styles -->
+<style>
+    article {
+        p { color: #999; align: left }
+    }
+    article {
+        p { color: #999;
+            align: left }
+    }
+    article {
+        p {
+            color: #999;
+            align: left }
+    }
+    article {
+        p { color: #999;
+            align: left
+        }
+    }
+    article {
+        p {
+            color: #999;
+            align: left
+        }
+    }
+    article {
+        p
+        {
+            color: #999;
+            align: left;
+        }
+    }
+</style>

From 8083d0a8e0eab5279332c8951e1b4fb0c0f7af9a Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Wed, 15 Feb 2023 01:11:52 +0100
Subject: [PATCH 04/45] Tidy up for next release

---
 README.md                  | 101 +++++++++--------
 djhtml/__main__.py         |   6 +
 djhtml/lines.py            |  21 ++--
 djhtml/modes.py            | 218 ++++++++++++++++++++++---------------
 djhtml/options.py          |   3 +-
 djhtml/tokens.py           |  11 +-
 noxfile.py                 |   2 +-
 tests/suite/paradoxes.html |  23 ++--
 8 files changed, 229 insertions(+), 156 deletions(-)

diff --git a/README.md b/README.md
index 7d9b2a0..a6e4c7f 100644
--- a/README.md
+++ b/README.md
@@ -51,10 +51,39 @@ This is what it will look like after processing by DjHTML:
 ```
 
 
+## New! Multi-line HTML elements
+
+As of version 3, DjHTML indents multi-line HTML elements and
+multi-line attribute values like this:
+
+```html
+<blockquote cite="John Lennon"
+            style="color: dimgray;
+                   font-style: italic;
+                   border-left: 5px solid gray;
+                  ">
+    It's weird not to be weird.
+</blockquote>
+```
+
+
+## New! Django middleware
+
+To automatically indent all the HTML responses from your Django web
+application, add the following to your settings file:
+
+    TABWIDTH = 4
+    MIDDLEWARE += ['djhtml.middleware.DjHTMLMiddleware']
+
+(Caution: when used in production, it is advised to use some kind of
+[caching](https://docs.djangoproject.com/en/stable/topics/cache/).)
+
+
 ## Installation
 
-DjHTML is compatible with all operating systems supported by Python.
-Install DjHTML with the following command:
+DjHTML requires Python 3.8 or higher and is compatible with all
+operating systems supported by Python. Install DjHTML with the
+following command:
 
     $ pip install djhtml
 
@@ -87,34 +116,6 @@ but no changes are actually made. All available options are given by
 `djthml -h` / `djthml --help`.
 
 
-## New! Multi-line HTML tag alignment
-
-As of version 3, DjHTML indents multi-line HTML tags and multi-line
-attribute values like this:
-
-```html
-<blockquote cite="John Lennon"
-            style="color: dimgray;
-                   font-style: italic;
-                   border-left: 5px solid gray;
-                  ">
-    It's weird not to be weird.
-</blockquote>
-```
-
-
-## New! Django middleware
-
-To automatically indent all the HTML responses from your Django web
-application, add the following to your settings file:
-
-    TABWIDTH = 4
-    MIDDLEWARE += ['djhtml.middleware.DjHTMLMiddleware']
-
-(Caution: when used in production, it is advised to use some kind of
-[caching](https://docs.djangoproject.com/en/stable/topics/cache/).)
-
-
 ## `fmt:off` and `fmt:on`
 
 You can exclude specific lines from being processed with the
@@ -235,23 +236,33 @@ Your feedback for improving DjHTML is very welcome!
 
 ## Development
 
-Use your preferred system for setting up a virtualenv, docker environment,
-or whatever else, then run the following:
+First of all, clone this repository:
 
-```sh
-python -m pip install -e '.[dev]'
-pre-commit install --install-hooks
-```
+    $ git clone https://github.com/rtts/djhtml
+    $ cd djhtml
 
-Tests can then be run quickly in that environment:
+Then, create a Python virtualenv and activate it:
 
-```sh
-python -m unittest discover -v
-```
+    $ python -m venv ~/.virtualenvs/djhtml
+    $ . ~/.virtualenvs/djhtml/bin/activate
 
-Or testing in all available supported environments and linting can be run
-with [`nox`](https://nox.thea.codes):
+Then, install the package in [development
+mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html)
+including the `dev` dependencies, and install the pre-commit hooks:
 
-```sh
-nox
-```
+    $ python -m pip install -e '.[dev]'
+    $ pre-commit install --install-hooks
+
+You can run the unittests with:
+
+    $ python -m unittest
+
+Or use [`nox`](https://nox.thea.codes) to test all supported Python
+interpreters:
+
+    $ nox
+
+Finally, to get a little insight into the tokenization step of the
+indenting algorithm, you can run DjHTML with the `-d` / `--debug`
+argument. You will see a Python representation of the tokens that are
+created.
diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index aa70951..6335db2 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -7,6 +7,12 @@
 standard output. Example usage:
 
     $ djhtml - < input.html > output.html
+
+Passing a directory name will recurse into the directory and format
+all files with typical extensions. For more fine-grained control of
+which files get processed, use external tools like find, xargs or
+pre-commit.
+
 """
 
 import sys
diff --git a/djhtml/lines.py b/djhtml/lines.py
index 0e67985..0434f2f 100644
--- a/djhtml/lines.py
+++ b/djhtml/lines.py
@@ -5,6 +5,11 @@ class Line:
     """
 
     def __init__(self, tokens=None, level=0, offset=0, ignore=False):
+        """
+        Lines are currently never instantiated with arguments, but
+        that doesn't mean they can't.
+
+        """
         self.tokens = tokens or []
         self.level = level
         self.offset = offset
@@ -12,7 +17,7 @@ def __init__(self, tokens=None, level=0, offset=0, ignore=False):
 
     def append(self, token):
         """
-        Append tokens to the line.
+        Append token to line.
 
         """
         self.tokens.append(token)
@@ -20,29 +25,29 @@ def append(self, token):
     @property
     def text(self):
         """
-        The text of this line without leading/trailing spaces.
+        The text of this line including the original
+        leading/trailing spaces.
 
         """
-        return "".join([str(token) for token in self.tokens]).strip()
+        return "".join([token.text for token in self.tokens])
 
     def indent(self, tabwidth):
         """
-        The final, indented text of this line. Make sure to set the level
-        and optionally offset before calling this method.
+        The final, indented text of this line.
 
         """
         if self.ignore:
-            return "".join([str(token) for token in self.tokens])
+            return self.text
         if self.text:
             spaces = tabwidth * self.level + self.offset
-            return " " * spaces + self.text
+            return " " * spaces + self.text.strip()
         return ""
 
     def __bool__(self):
         return bool(self.tokens)
 
     def __len__(self):
-        return len(self.text)
+        return len(self.text.strip())
 
     def __repr__(self):
         kwargs = ""
diff --git a/djhtml/modes.py b/djhtml/modes.py
index 7cdc089..fd1c2ea 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -4,49 +4,29 @@
 from .tokens import Token
 
 
-class DjTXT:
+class BaseMode:
     """
-    Mode for indenting Django/Jinja template tags.
-    Also serves as the base class for the other modes:
-
-    - DjHTML
-    - DjCSS
-    - DjJS
+    Base class for the different modes.
 
     """
 
-    RAW_TOKENS = [
-        r"\n",
-        r"{%[-\+]?.*?[-\+]?%}",
-        r"{#.*?#}",
-        r"{#",
-        r"{{.*?}}",
-    ]
-    CLOSING_AND_OPENING_TAGS = [
-        "elif",
-        "else",
-        "empty",
-        "plural",
-    ]
-    COMMENT_TAGS = [
-        "comment",
-        "verbatim",
-        "raw",
-    ]
-    AMBIGUOUS_BLOCK_TAGS = {
-        # token_name: (regex_if_block, regex_if_not_block)
-        "set": (None, " = "),
-        "video": (" as ", None),
-    }
-    FMT_ON = r"{# fmt:on #}"
-    FMT_OFF = r"{# fmt:off #}"
-    OPENING_TAG = r"{%[-\+]? *(\w+).*?[-\+]?%}"
-
     def __init__(self, source="", return_mode=None):
+        """
+        Instantiate with source text before calling indent(), or
+        with the return_mode when invoked from within another mode.
+
+        """
+        assert type(self) is not BaseMode
+        assert source or return_mode
+
         self.source = source
         self.return_mode = return_mode or self
         self.token_re = compile_re(self.RAW_TOKENS)
-        self.offsets = {"relative": 0, "absolute": 0}
+
+        # To keep track of the current offsets.
+        self.offsets = dict(relative=0, absolute=0)
+
+        # To be used as a stack by some modes.
         self.previous_offsets = []
 
     def indent(self, tabwidth):
@@ -58,15 +38,75 @@ def indent(self, tabwidth):
         self.parse()
         return "\n".join([line.indent(tabwidth) for line in self.lines])
 
+    def tokenize(self):
+        """
+        Split the source text into tokens and place them on lines.
+
+        How text is split into raw tokens is defined by the regexes
+        of each mode. For each raw token, the create_token() method of
+        the mode is called.
+
+        Some tokens, such a <style> or <script> tags in HTML, can
+        switch to a different mode by returning a new instance of that
+        mode alongside the token.
+
+        """
+        self.lines = []
+        line = Line()
+        mode = self
+        src = self.source
+
+        while True:
+            try:
+                # Split the source at the first occurrence of one of
+                # the current mode's raw tokens.
+                head, raw_token, tail = mode.token_re.split(src, maxsplit=1)
+
+            except ValueError:
+                # We've reached the final line!
+                if src:
+                    token, _ = mode.create_token(src, "", line)
+                    line.append(token)
+                self.lines.append(line)
+                break
+
+            if head:
+                # Create a token from the head.
+                token, mode = mode.create_token(head, raw_token + tail, line)
+                line.append(token)
+
+            if raw_token == "\n":
+                self.lines.append(line)
+                line = Line()
+
+            else:
+                # Create a token from the tail
+                token, mode = mode.create_token(raw_token, tail, line)
+                line.append(token)
+
+            # Set the new source to the old tail for the next iteration.
+            src = tail
+
     def parse(self):
         """
         You found the top-secret indenting algorithm!
 
+        Once the source has been split into tokens, the indentation is
+        solely determined by the attributes of the tokens. This makes
+        the algorithm independent of the language (HTML, CSS, JS), and
+        thereby accomodates different languages used interchangeably.
+
         """
         stack = []
 
         def mode_in_stack(mode):
+            """
+            Helper function to see if a token from a specific mode
+            is in the stack.
+
+            """
             if stack[-1].mode is DjTXT:
+                # See paradoxes.html and issue #17
                 return False
             for token in stack:
                 if token.mode is mode:
@@ -78,33 +118,44 @@ def mode_in_stack(mode):
             for token in line.tokens:
                 opening_token = None
                 if stack:
-                    # When a dedenting token is found, match it with
-                    # the token at the top of the stack. To find out
-                    # what each statement does, uncomment it and see
-                    # which unittests fail.
+
+                    # When a dedenting token is found, pop the
+                    # corresponding opening token from the stack.
                     if token.dedents:
                         if stack[-1].mode is token.mode:
                             opening_token = stack.pop()
                             if stack and (opening_token.is_double or token.is_double):
                                 opening_token = stack.pop()
+
+                        # Error: the opening token is from a different
+                        # mode. Pop the stack until the correct
+                        # opening token is found.
                         elif mode_in_stack(token.mode):
                             opening_token = stack.pop()
                             while opening_token.mode is not token.mode:
                                 opening_token = stack.pop()
+
+                        # Error: there are no tokens in the stack of
+                        # the same mode. Set the line level to a sane
+                        # value.
                         elif first_token:
                             line.level = stack[-1].level + 1
 
-                        # Dedent!
+                        # Success! If the dedenting token is first in
+                        # line, set the line level to the level of the
+                        # opening token.
                         if first_token and opening_token:
                             line.level = opening_token.level
 
+                    # For non-dedenting tokens, set the line one level
+                    # higher than the opening token's level.
                     else:
-                        # Indent!
                         if token.is_double and stack[-1].is_double:
                             opening_token = stack.pop()
                         if stack and first_token:
                             line.level = stack[-1].level + 1
 
+                # Adjust line level according to the token's offsets.
                 if first_token:
                     line.level = line.level + token.relative
                     line.offset = token.absolute
@@ -118,53 +169,49 @@ def mode_in_stack(mode):
                 if token.text.strip():
                     first_token = False
 
-    def tokenize(self):
-        """
-        Split the source text into tokens and place them on lines.
-
-        """
-        self.lines = []
-        line = Line()
-        mode = self
-        src = self.source
-
-        while True:
-            try:
-                # Split the source at the first occurrence of one of
-                # the current mode's raw tokens.
-                head, raw_token, tail = mode.token_re.split(src, maxsplit=1)
+    def debug(self):
+        self.tokenize()
+        self.parse()
+        return "\n".join([repr(line) for line in self.lines])
 
-            except ValueError:
-                # We've reached the final line!
-                if src:
-                    token, _ = mode.create_token(src, "", line)
-                    line.append(token)
-                self.lines.append(line)
-                break
 
-            if head:
-                # Create a token from the head.
-                token, mode = mode.create_token(head, raw_token + tail, line)
-                line.append(token)
+class DjTXT(BaseMode):
+    """
+    Mode for indenting text containing Django/Jinja template tags.
 
-            if raw_token == "\n":
-                self.lines.append(line)
-                line = Line()
+    This mode is special because all the other modes inherit from it,
+    because all other modes can contain Django/Jinja template tags.
 
-            else:
-                # Create a token from the tail
-                token, mode = mode.create_token(raw_token, tail, line)
-                line.append(token)
+    """
 
-            # Set the new source to the old tail for the next iteration.
-            src = tail
+    RAW_TOKENS = [
+        r"\n",
+        r"{%[-\+]?.*?[-\+]?%}",
+        r"{#.*?#}",
+        r"{#",
+        r"{{.*?}}",
+    ]
+    CLOSING_AND_OPENING_TAGS = [
+        "elif",
+        "else",
+        "empty",
+        "plural",
+    ]
+    COMMENT_TAGS = [
+        "comment",
+        "verbatim",
+        "raw",
+    ]
+    AMBIGUOUS_BLOCK_TAGS = {
+        # token_name: (regex_if_block, regex_if_not_block)
+        "set": (None, " = "),
+        "video": (" as ", None),
+    }
+    FMT_ON = r"{# fmt:on #}"
+    FMT_OFF = r"{# fmt:off #}"
+    OPENING_TAG = r"{%[-\+]? *(\w+).*?[-\+]?%}"
 
     def create_token(self, raw_token, src, line):
-        """
-        Given a raw token string, return a single token and the
-        next mode.
-
-        """
         mode = self
 
         if tag := re.match(self.OPENING_TAG, raw_token):
@@ -212,17 +259,10 @@ def _has_closing_token(self, name, raw_token, src):
                 return not re.search(regex[1], raw_token)
         return True
 
-    def debug(self):
-        self.tokenize()
-        self.parse()
-        return "\n".join([repr(line) for line in self.lines])
-
 
 class DjHTML(DjTXT):
     """
-    This mode is the entrypoint of DjHTML. Usage:
-
-    >>> DjHTML(input_string).indent(tabwidth=4)
+    Mode for indenting HTML.
 
     """
 
diff --git a/djhtml/options.py b/djhtml/options.py
index 9fee2b1..aaa48d1 100644
--- a/djhtml/options.py
+++ b/djhtml/options.py
@@ -3,6 +3,7 @@
 
     import options
     print(f"Tabwidth is {options.tabwidth}")
+
 """
 
 import argparse
@@ -38,7 +39,7 @@
     nargs="+",
     help="input filenames (either paths or directories)",
 )
-parser.add_argument("-d", "--debug", action="store_true", help=argparse.SUPPRESS)
+parser.add_argument("-d", "--debug", action="store_true", help="debug mode")
 parser.add_argument("-i", "--in-place", action="store_true", help=argparse.SUPPRESS)
 
 # Parse arguments and assign attributes to self
diff --git a/djhtml/tokens.py b/djhtml/tokens.py
index 317feee..9304250 100644
--- a/djhtml/tokens.py
+++ b/djhtml/tokens.py
@@ -13,6 +13,14 @@ class _Base:
         def __init__(
             self, text, *, mode, level=0, relative=0, absolute=0, ignore=False
         ):
+            """
+            Tokens must have a text and a mode class. The level
+            represents the line level of opening tokens and is set
+            afterwards by the parser. This violates the principle of
+            encapsulation, but makes sense because the line level can
+            only be determined after the tokenization is complete.
+
+            """
             self.text = text
             self.mode = mode
             self.level = level
@@ -20,9 +28,6 @@ def __init__(
             self.absolute = absolute
             self.ignore = ignore
 
-        def __str__(self):
-            return self.text
-
         def __repr__(self):
             kwargs = f", mode={self.mode.__name__}"
             for attr in ["level", "relative", "absolute", "ignore"]:
diff --git a/noxfile.py b/noxfile.py
index 0942baf..13b1406 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -3,4 +3,4 @@
 
 @nox.session(python=["3.8", "3.9", "3.10", "3.11"])
 def tests(session):
-    session.run("python", "-m", "unittest", "discover", "-v")
+    session.run("python", "-m", "unittest")
diff --git a/tests/suite/paradoxes.html b/tests/suite/paradoxes.html
index 653231e..3c8eae0 100644
--- a/tests/suite/paradoxes.html
+++ b/tests/suite/paradoxes.html
@@ -1,13 +1,15 @@
 <!-- Common pattern in django templates -->
-{% if %}
-    <a>
-{% else %}
-    <a>
-{% endif %}
-.
-{% if %}
-    </a>
-{% endif %}
+<div>
+    {% if some_condition %}
+        <a href="{% url a %}">
+    {% elif some_other condition %}
+        <a href="{% url b %}">
+    {% endif %}
+    .
+    {% if some_condition or some_other_condition %}
+        </a>
+    {% endif %}
+</div>
 
 
 <!-- The literal example from issue #17 -->
@@ -43,3 +45,6 @@
     console.log("Time to ditch jQuery!")
     });
 </script>
+
+
+<!-- Final note: all other malstructured source files will result in undefined behavior, but not in errors -->

From 4c7b1f19c952906a8ba6b9fb80de0f8facb0613b Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Wed, 15 Feb 2023 10:31:13 +0100
Subject: [PATCH 05/45] Add `{% placeholder %}` to ambiguous block tags

Closes #78
---
 djhtml/modes.py         |  1 +
 tests/suite/django.html | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index fd1c2ea..8a46d45 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -206,6 +206,7 @@ class DjTXT(BaseMode):
         # token_name: (regex_if_block, regex_if_not_block)
         "set": (None, " = "),
         "video": (" as ", None),
+        "placeholder": (" or ", None),
     }
     FMT_ON = r"{# fmt:on #}"
     FMT_OFF = r"{# fmt:off #}"
diff --git a/tests/suite/django.html b/tests/suite/django.html
index 80b8fba..f31857f 100644
--- a/tests/suite/django.html
+++ b/tests/suite/django.html
@@ -136,3 +136,30 @@ <h1>Welcome, {{ request.user }}</h1>
     {{ '</b>
 if condition }}
 more text
+
+
+<!-- The example from issue #78 -->
+{% block main %}
+    <section>
+        <div>
+            <div>
+                {% placeholder "placeholder1" %}
+            </div>
+            <div>
+                {% placeholder "placeholder2" or %}
+                    <img src="" alt="" />
+                {% endplaceholder %}
+            </div>
+        </div>
+    </section>
+    <section>
+        <div>
+            {% placeholder "placeholder3" %}
+        </div>
+    </section>
+    <section>
+        <div>
+            {% placeholder "placeholder4" %}
+        </div>
+    </section>
+{% endblock %}

From 0ec83355f66dbaee8b964c4bb5074fdfe771e649 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Thu, 16 Feb 2023 09:13:36 +0100
Subject: [PATCH 06/45] Upgrade Black to 23.1.0

---
 .pre-commit-config.yaml | 2 +-
 djhtml/modes.py         | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a4cd384..8b0fc01 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
   hooks:
   - id: isort
 - repo: https://github.com/psf/black
-  rev: 22.12.0
+  rev: 23.1.0
   hooks:
   - id: black
 - repo: https://github.com/pycqa/flake8
diff --git a/djhtml/modes.py b/djhtml/modes.py
index 8a46d45..e142a5d 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -118,7 +118,6 @@ def mode_in_stack(mode):
             for token in line.tokens:
                 opening_token = None
                 if stack:
-
                     # When a dedenting token is found, pop the
                     # corresponding opening token from the stack.
                     if token.dedents:
@@ -218,7 +217,11 @@ def create_token(self, raw_token, src, line):
         if tag := re.match(self.OPENING_TAG, raw_token):
             name = tag.group(1)
             if name in self.COMMENT_TAGS:
-                token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True,), Comment(
+                token, mode = Token.Open(
+                    raw_token,
+                    mode=DjTXT,
+                    ignore=True,
+                ), Comment(
                     r"{%[-\+]? *end" + name + r"(?: .*?|)%}",
                     mode=DjTXT,
                     return_mode=self,

From 1fd02b2c0ddeff79a008386e6e8246d5c91780fd Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Thu, 16 Feb 2023 20:31:08 +0100
Subject: [PATCH 07/45] Correct handling of single-line JavaScript comments

This closes the second part of #79, not the first part.
---
 djhtml/modes.py     |  5 +++--
 tests/suite/js.html | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index e142a5d..fe018be 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -396,13 +396,14 @@ class DjJS(DjTXT):
 
     RAW_TOKENS = DjTXT.RAW_TOKENS + [
         r"</script>",
+        r"\\.?",
+        r"//.*",
+        r"/\*",
         r'".*?"',
         r"'.*?'",
         r"`.*?`",
         r"`",
         r"[\{\[\(\)\]\}]",
-        r"//.*",
-        r"/\*",
     ]
 
     def create_token(self, raw_token, src, line):
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 053f135..33ad388 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -285,3 +285,18 @@
         }
     </script>
 </head>
+
+
+<!-- Single-line comments -->
+<script>
+    // The following does not contain a comment
+    wsHost = (/^wss?:\/\//i.test(me._host) ? '' : me._protocol) + me._host;
+
+    foo; // This is a comment
+    bar //This too
+    baz//Even this
+
+    foo // function() {
+    bar //     var a = 1;
+    baz // }
+</script>

From bae7427f07e7376ae992fa7be42584947f09a893 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Thu, 16 Feb 2023 22:40:30 +0100
Subject: [PATCH 08/45] JavaScript multi-line variable assignment

Closes #79
---
 djhtml/modes.py     | 37 ++++++++++++++++++++++++++++++++++++-
 tests/suite/js.html | 30 ++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index fe018be..05df7f8 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -404,15 +404,38 @@ class DjJS(DjTXT):
         r"`.*?`",
         r"`",
         r"[\{\[\(\)\]\}]",
+        r"var ",
+        r"let ",
+        r"const ",
+        r";",
     ]
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.haskell = False
+        self.haskell_re = re.compile(r" *, [^ ]+ *=")
+        self.variable_re = re.compile(r" *[^ ]+ *=")
+
     def create_token(self, raw_token, src, line):
         mode = self
 
+        if (
+            not len(line)
+            and not self.haskell_re.match(raw_token)
+            and not self.variable_re.match(raw_token)
+        ):
+            self.offsets["absolute"] = 0
+
         if raw_token in "{[(":
+            self.previous_offsets.append(self.offsets["absolute"])
+            self.offsets["absolute"] = 0
             token = Token.Open(raw_token, mode=DjJS, **self.offsets)
         elif raw_token in ")]}":
             token = Token.Close(raw_token, mode=DjJS, **self.offsets)
+            try:
+                self.offsets["absolute"] = self.previous_offsets.pop()
+            except IndexError:
+                self.offsets["absolute"] = 0
         elif raw_token == "`":
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 "`", mode=DjJS, return_mode=self
@@ -421,12 +444,24 @@ def create_token(self, raw_token, src, line):
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 r"\*/", mode=DjJS, return_mode=self
             )
-        elif not line and raw_token.lstrip().startswith("."):
+        elif not len(line) and raw_token.lstrip().startswith("."):
             self.offsets["relative"] = 1
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif raw_token.lstrip().startswith(("case ", "default:")):
             token = Token.OpenDouble(raw_token, mode=DjJS)
             return token, mode
+        elif not len(line) and raw_token in ["var ", "let ", "const "]:
+            self.haskell = False
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
+            self.offsets["absolute"] = len(raw_token)
+        elif not len(line) and not self.haskell and self.haskell_re.match(raw_token):
+            self.haskell = True
+            self.offsets["absolute"] -= 2
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
+        elif raw_token == ";":
+            self.haskell = False
+            self.offsets["absolute"] = 0
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif raw_token == "</script>":
             token, mode = (
                 Token.Close(raw_token, mode=self.return_mode.__class__),
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 33ad388..2afdd22 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -300,3 +300,33 @@
     bar //     var a = 1;
     baz // }
 </script>
+
+
+<!-- Multi-line variable assignment -->
+<script>
+    var a = 1,
+        b = 2,
+        c = 3;
+    let a = 1
+      , b = 2
+      , c = 3
+    ;
+    const a = 1
+        , b = 2
+        , c = 3;
+    d = 4
+
+    const func = function() {
+        a = 1
+        b = 2
+        c = 3
+    }
+    const func1 = function() { return 1 },
+          func2 = function() {
+        const a = 1,
+              b = 2,
+              c = 3
+        return b
+    },
+          func3 = function() { return 3 };
+</script>

From ce41ad1bce247854d13180dd1964de482940783e Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Thu, 16 Feb 2023 22:43:21 +0100
Subject: [PATCH 09/45] Fix AssertionError on empty files

---
 djhtml/modes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 05df7f8..3608d18 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -10,14 +10,14 @@ class BaseMode:
 
     """
 
-    def __init__(self, source="", return_mode=None):
+    def __init__(self, source=None, return_mode=None):
         """
         Instantiate with source text before calling indent(), or
         with the return_mode when invoked from within another mode.
 
         """
         assert type(self) is not BaseMode
-        assert source or return_mode
+        assert source is not None or return_mode
 
         self.source = source
         self.return_mode = return_mode or self

From a877f30479d904be08a6cda6b1e55f886b753162 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 11:43:19 +0100
Subject: [PATCH 10/45] Better handling of multi-line CSS statements

---
 djhtml/lines.py      | 10 ++++++++++
 djhtml/modes.py      | 37 ++++++++++++-------------------------
 tests/suite/css.html | 43 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 25 deletions(-)

diff --git a/djhtml/lines.py b/djhtml/lines.py
index 0434f2f..5db28eb 100644
--- a/djhtml/lines.py
+++ b/djhtml/lines.py
@@ -31,6 +31,16 @@ def text(self):
         """
         return "".join([token.text for token in self.tokens])
 
+    @property
+    def first_token(self):
+        """
+        The first non-empty token of this line.
+
+        """
+        for token in self.tokens:
+            if token.text.strip():
+                return token
+
     def indent(self, tabwidth):
         """
         The final, indented text of this line.
diff --git a/djhtml/modes.py b/djhtml/modes.py
index 3608d18..862391b 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -335,44 +335,31 @@ class DjCSS(DjTXT):
         r"</style>",
         r"[\{\(\)\}]",
         r"/\*",
-        r"[\w-]+: ",
+        r'".+"',
+        r"'.+'",
+        r"[\w-]+:",
         r";",
     ]
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.inside_block = False
-
     def create_token(self, raw_token, src, line):
         mode = self
 
         if raw_token in "{(":
-            self.previous_offsets.append(self.offsets["relative"])
             token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
-            self.offsets["relative"] = 0
             if raw_token == "{":
-                self.inside_block = True
-        elif raw_token in "})":
-            if raw_token == "}" and self.offsets["absolute"]:
                 self.offsets["absolute"] = 0
-                self.offsets["relative"] = 0
-                token = Token.CloseDouble(raw_token, mode=DjCSS, **self.offsets)
-            else:
-                token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
-                try:
-                    self.offsets["relative"] = self.previous_offsets.pop()
-                except IndexError:
-                    self.offsets["relative"] = 0
+                if first_token := line.first_token:
+                    first_token.absolute = 0
+        elif raw_token in "})":
             if raw_token == "}":
-                self.inside_block = False
-        elif self.inside_block and raw_token.endswith(": "):
-            token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
-            self.offsets["relative"] = -1
-            self.offsets["absolute"] = len(raw_token)
+                self.offsets["absolute"] = 0
+            token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
+        elif raw_token.endswith(":"):
+            token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
+            self.offsets["absolute"] = len(line) + len(raw_token) + 1
         elif raw_token == ";":
-            self.offsets["relative"] = 0
             self.offsets["absolute"] = 0
-            token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
+            token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
         elif raw_token == "/*":
             token, mode = Token.Open(raw_token, mode=DjCSS, ignore=True), Comment(
                 r"\*/", mode=DjCSS, return_mode=self
diff --git a/tests/suite/css.html b/tests/suite/css.html
index 6c71e42..5799dbc 100644
--- a/tests/suite/css.html
+++ b/tests/suite/css.html
@@ -110,3 +110,46 @@
         }
     }
 </style>
+
+
+<!-- Selector weirdness -->
+<style>
+    blockquote:before {
+        content: "text-align: right";
+    }
+    @media(text-align: right) {
+        a:hover, a:active {
+            text-decoration:none
+        }
+        p::first-line {
+            color: red;
+        }
+    }
+</style>
+
+
+<!-- Nested SCSS with multi-line statements -->
+<style>
+    a:hover, a:active {
+        foo: bar,
+             baz;
+
+        a:hover, a:active {
+            foo: bar
+                 baz
+        }
+    }
+</style>
+
+
+<!-- The difference between multi-line statements and nested selectors -->
+<style>
+    body {
+        font-family: Arial,
+                     Helvetica;
+        a:hover,
+        a:active {
+            color: red;
+        }
+    }
+</style>

From 79c61f9d3655089b1370ab3259a04939866db10f Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 13:02:31 +0100
Subject: [PATCH 11/45] Better handling of multi-line JS variable assignments

---
 djhtml/modes.py     | 37 ++++++++++++++++++++++---------------
 tests/suite/js.html | 41 +++++++++++++++++++++++------------------
 2 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 862391b..fc362ef 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -394,7 +394,6 @@ class DjJS(DjTXT):
         r"var ",
         r"let ",
         r"const ",
-        r";",
     ]
 
     def __init__(self, *args, **kwargs):
@@ -402,16 +401,18 @@ def __init__(self, *args, **kwargs):
         self.haskell = False
         self.haskell_re = re.compile(r" *, [^ ]+ *=")
         self.variable_re = re.compile(r" *[^ ]+ *=")
+        self.previous_line_ended_with_comma = False
 
     def create_token(self, raw_token, src, line):
         mode = self
 
-        if (
-            not len(line)
-            and not self.haskell_re.match(raw_token)
-            and not self.variable_re.match(raw_token)
-        ):
-            self.offsets["absolute"] = 0
+        # Reset absolute offset in almost all cases
+        if not len(line) and not self.haskell_re.match(raw_token):
+            if (
+                not self.variable_re.match(raw_token)
+                or not self.previous_line_ended_with_comma
+            ):
+                self.offsets["absolute"] = 0
 
         if raw_token in "{[(":
             self.previous_offsets.append(self.offsets["absolute"])
@@ -437,18 +438,19 @@ def create_token(self, raw_token, src, line):
         elif raw_token.lstrip().startswith(("case ", "default:")):
             token = Token.OpenDouble(raw_token, mode=DjJS)
             return token, mode
-        elif not len(line) and raw_token in ["var ", "let ", "const "]:
+        elif raw_token in ["var ", "let ", "const "]:
             self.haskell = False
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
-            self.offsets["absolute"] = len(raw_token)
-        elif not len(line) and not self.haskell and self.haskell_re.match(raw_token):
+            self.offsets["absolute"] = len(line.text.lstrip()) + len(raw_token)
+        elif (
+            not len(line)
+            and not self.haskell
+            and not self.previous_line_ended_with_comma
+            and self.haskell_re.match(raw_token)
+        ):
             self.haskell = True
             self.offsets["absolute"] -= 2
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
-        elif raw_token == ";":
-            self.haskell = False
-            self.offsets["absolute"] = 0
-            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif raw_token == "</script>":
             token, mode = (
                 Token.Close(raw_token, mode=self.return_mode.__class__),
@@ -458,9 +460,14 @@ def create_token(self, raw_token, src, line):
             token, mode = super().create_token(raw_token, src, line)
 
         # Reset relative offset after creating first token in line.
-        if not line:
+        if not len(line):
             self.offsets["relative"] = 0
 
+        if raw_token.rstrip().endswith(","):
+            self.previous_line_ended_with_comma = True
+        else:
+            self.previous_line_ended_with_comma = False
+
         return token, mode
 
 
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 2afdd22..0e560f0 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -307,26 +307,31 @@
     var a = 1,
         b = 2,
         c = 3;
-    let a = 1
-      , b = 2
-      , c = 3
-    ;
+    var foo = bar; let a = 1,
+                       b = 2,
+                       c = 3
+
+    // Haskell-style indentation
     const a = 1
         , b = 2
-        , c = 3;
-    d = 4
+        , c = 3
 
-    const func = function() {
-        a = 1
-        b = 2
-        c = 3
-    }
-    const func1 = function() { return 1 },
-          func2 = function() {
-        const a = 1,
-              b = 2,
-              c = 3
-        return b
+    // Notice the first line is the same
+    const a = 1
+    b = 2
+    c = 3
+
+    // Yes, this makes sense
+    const func1 = function() {
+        return foo
     },
-          func3 = function() { return 3 };
+          func2 = function() { return 2 },
+          func3 = function() { return 3 }
+
+    // We are not supporting madness like this
+    const a = 1,
+    {% if %}
+        b = 2,
+    {% endif %}
+    c = 3;
 </script>

From b3aaa160d0e593722e054e038519cfae4c6fd2bd Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 13:12:46 +0100
Subject: [PATCH 12/45] Fix CSS multi-line with prior statements

---
 djhtml/modes.py      |  2 +-
 tests/suite/css.html | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index fc362ef..6cfc4fd 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -356,7 +356,7 @@ def create_token(self, raw_token, src, line):
             token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
         elif raw_token.endswith(":"):
             token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
-            self.offsets["absolute"] = len(line) + len(raw_token) + 1
+            self.offsets["absolute"] = len(line.text.lstrip()) + len(raw_token) + 1
         elif raw_token == ";":
             self.offsets["absolute"] = 0
             token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
diff --git a/tests/suite/css.html b/tests/suite/css.html
index 5799dbc..d490be5 100644
--- a/tests/suite/css.html
+++ b/tests/suite/css.html
@@ -153,3 +153,16 @@
         }
     }
 </style>
+
+
+<!-- Multi-line statements with prior statements -->
+<style>
+    body {
+        color: green; font-family: Arial,
+                                   Helvetica;
+
+        /* Spaces within lines are always preserved */
+        color: red;   font-family: Arial,
+                                   Helvetica;
+    }
+</style>

From bd324e6b97b2cb8097be542c59feec52ea5b9e48 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 14:30:33 +0100
Subject: [PATCH 13/45] Fix multi-line HTML elements with preceding elements

---
 djhtml/lines.py       | 22 ++++++++++++++++++----
 djhtml/modes.py       | 33 +++++++++++++++++----------------
 tests/suite/html.html | 12 ++++++------
 3 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/djhtml/lines.py b/djhtml/lines.py
index 5db28eb..5f69777 100644
--- a/djhtml/lines.py
+++ b/djhtml/lines.py
@@ -41,6 +41,16 @@ def first_token(self):
             if token.text.strip():
                 return token
 
+    @property
+    def indents(self):
+        """
+        Whether this line has more opening than closing tokens.
+
+        """
+        return len([token for token in self.tokens if token.indents]) > len(
+            [token for token in self.tokens if token.dedents]
+        )
+
     def indent(self, tabwidth):
         """
         The final, indented text of this line.
@@ -53,11 +63,15 @@ def indent(self, tabwidth):
             return " " * spaces + self.text.strip()
         return ""
 
-    def __bool__(self):
-        return bool(self.tokens)
-
     def __len__(self):
-        return len(self.text.strip())
+        """
+        The length of the line (so far), excluding the whitespace
+        at the beginning. Be careful calling len() because it might
+        result in trailing spaces being counted that will be removed
+        by indent().
+
+        """
+        return len(self.text.lstrip())
 
     def __repr__(self):
         kwargs = ""
diff --git a/djhtml/modes.py b/djhtml/modes.py
index 6cfc4fd..c8f3c0d 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -301,7 +301,7 @@ def create_token(self, raw_token, src, line):
 
         if raw_token == "<":
             if tag := re.match(r"([\w\-\.:]+)", src):
-                token, mode = Token.Open(raw_token, mode=DjHTML), InsideHTMLTag(
+                token, mode = Token.Text(raw_token, mode=DjHTML), InsideHTMLTag(
                     tag[1], line, self
                 )
             else:
@@ -356,7 +356,7 @@ def create_token(self, raw_token, src, line):
             token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
         elif raw_token.endswith(":"):
             token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
-            self.offsets["absolute"] = len(line.text.lstrip()) + len(raw_token) + 1
+            self.offsets["absolute"] = len(line) + len(raw_token) + 1
         elif raw_token == ";":
             self.offsets["absolute"] = 0
             token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
@@ -407,7 +407,7 @@ def create_token(self, raw_token, src, line):
         mode = self
 
         # Reset absolute offset in almost all cases
-        if not len(line) and not self.haskell_re.match(raw_token):
+        if not line and not self.haskell_re.match(raw_token):
             if (
                 not self.variable_re.match(raw_token)
                 or not self.previous_line_ended_with_comma
@@ -432,7 +432,7 @@ def create_token(self, raw_token, src, line):
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 r"\*/", mode=DjJS, return_mode=self
             )
-        elif not len(line) and raw_token.lstrip().startswith("."):
+        elif not line and raw_token.lstrip().startswith("."):
             self.offsets["relative"] = 1
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif raw_token.lstrip().startswith(("case ", "default:")):
@@ -441,9 +441,9 @@ def create_token(self, raw_token, src, line):
         elif raw_token in ["var ", "let ", "const "]:
             self.haskell = False
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
-            self.offsets["absolute"] = len(line.text.lstrip()) + len(raw_token)
+            self.offsets["absolute"] = len(line) + len(raw_token)
         elif (
-            not len(line)
+            not line
             and not self.haskell
             and not self.previous_line_ended_with_comma
             and self.haskell_re.match(raw_token)
@@ -460,7 +460,7 @@ def create_token(self, raw_token, src, line):
             token, mode = super().create_token(raw_token, src, line)
 
         # Reset relative offset after creating first token in line.
-        if not len(line):
+        if not line:
             self.offsets["relative"] = 0
 
         if raw_token.rstrip().endswith(","):
@@ -505,7 +505,9 @@ def __init__(self, tagname, line, return_mode):
         self.return_mode = return_mode
         self.token_re = compile_re(self.RAW_TOKENS)
         self.inside_attr = False
-        self.offsets = dict(relative=-1, absolute=len(line) + len(tagname) + 2)
+        self.offsets = dict(
+            relative=-1 if line.indents else 0, absolute=len(line) + len(tagname) + 2
+        )
         self.previous_offsets = []
 
         # Pff...
@@ -514,10 +516,9 @@ def __init__(self, tagname, line, return_mode):
     def create_token(self, raw_token, src, line):
         mode = self
 
-        if line:
-            self.additional_offset += len(raw_token)
-        else:
+        if not line:
             self.additional_offset = 0
+        self.additional_offset += len(raw_token)
 
         if "text/template" in raw_token:
             self.tagname = ""
@@ -534,21 +535,21 @@ def create_token(self, raw_token, src, line):
                 self.offsets["absolute"] += self.additional_offset
                 token = Token.Text(raw_token, mode=InsideHTMLTag, **self.offsets)
         elif not self.inside_attr and raw_token == "/>":
-            token, mode = Token.Close(raw_token, mode=DjHTML), self.return_mode
+            token, mode = Token.Text(raw_token, mode=DjHTML), self.return_mode
         elif not self.inside_attr and raw_token == ">":
             if self.tagname.lower() in DjHTML.IGNORE_TAGS:
-                token, mode = Token.Close(raw_token, mode=DjHTML), self.return_mode
+                token, mode = Token.Text(raw_token, mode=DjHTML), self.return_mode
             elif self.tagname == "style":
-                token, mode = Token.CloseAndOpen(raw_token, mode=DjHTML), DjCSS(
+                token, mode = Token.Open(raw_token, mode=DjHTML), DjCSS(
                     return_mode=self.return_mode
                 )
             elif self.tagname == "script":
-                token, mode = Token.CloseAndOpen(raw_token, mode=DjHTML), DjJS(
+                token, mode = Token.Open(raw_token, mode=DjHTML), DjJS(
                     return_mode=self.return_mode
                 )
             else:
                 token, mode = (
-                    Token.CloseAndOpen(raw_token, mode=DjHTML),
+                    Token.Open(raw_token, mode=DjHTML),
                     self.return_mode,
                 )
         else:
diff --git a/tests/suite/html.html b/tests/suite/html.html
index cf13b31..a47ab81 100644
--- a/tests/suite/html.html
+++ b/tests/suite/html.html
@@ -243,7 +243,7 @@ <h1>Title</h1>
 </a>
 
 
-<!-- The example from issue #50
+<!-- The example from issue #50 -->
 <img src="/some/long/path"
      id="1">
 <blockquote class="dishy"
@@ -313,11 +313,11 @@ <h1>Title</h1>
 
 
 <!-- Multi-line attribute value with additional elements/attributes -->
-<div><blockquote id="1" id="2"
-                 id="3" cite="
-                              Wikipedia
-                             "
-                 id="4"/></div>
+<div>text<blockquote id="1" id="2"
+                     id="3" cite="
+                                  Wikipedia
+                                 "
+                     id="4"/></div>
 
 
 <!-- The example from the README -->

From de41a063d5a05f6202237e275785ac919ed5f4ab Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 15:33:20 +0100
Subject: [PATCH 14/45] Expect a space between CSS properties and values

This doesn't mean a space is always needed, but it does mean a space
is needed if the value is multi-line. Otherwise, nested selectors with
pseudo-classes cannot be distinguished from multi-line statements.
---
 djhtml/lines.py      | 15 ++-------------
 djhtml/modes.py      |  8 +++-----
 tests/suite/css.html |  8 ++++++++
 3 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/djhtml/lines.py b/djhtml/lines.py
index 5f69777..010f0d9 100644
--- a/djhtml/lines.py
+++ b/djhtml/lines.py
@@ -31,16 +31,6 @@ def text(self):
         """
         return "".join([token.text for token in self.tokens])
 
-    @property
-    def first_token(self):
-        """
-        The first non-empty token of this line.
-
-        """
-        for token in self.tokens:
-            if token.text.strip():
-                return token
-
     @property
     def indents(self):
         """
@@ -58,9 +48,8 @@ def indent(self, tabwidth):
         """
         if self.ignore:
             return self.text
-        if self.text:
-            spaces = tabwidth * self.level + self.offset
-            return " " * spaces + self.text.strip()
+        if text := self.text.strip():
+            return " " * (tabwidth * self.level + self.offset) + text
         return ""
 
     def __len__(self):
diff --git a/djhtml/modes.py b/djhtml/modes.py
index c8f3c0d..5cbebb6 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -337,7 +337,7 @@ class DjCSS(DjTXT):
         r"/\*",
         r'".+"',
         r"'.+'",
-        r"[\w-]+:",
+        r"[\w-]+: ",
         r";",
     ]
 
@@ -348,15 +348,13 @@ def create_token(self, raw_token, src, line):
             token = Token.Open(raw_token, mode=DjCSS, **self.offsets)
             if raw_token == "{":
                 self.offsets["absolute"] = 0
-                if first_token := line.first_token:
-                    first_token.absolute = 0
         elif raw_token in "})":
             if raw_token == "}":
                 self.offsets["absolute"] = 0
             token = Token.Close(raw_token, mode=DjCSS, **self.offsets)
-        elif raw_token.endswith(":"):
+        elif raw_token.endswith(": "):
             token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
-            self.offsets["absolute"] = len(line) + len(raw_token) + 1
+            self.offsets["absolute"] = len(line) + len(raw_token)
         elif raw_token == ";":
             self.offsets["absolute"] = 0
             token = Token.Text(raw_token, mode=DjCSS, **self.offsets)
diff --git a/tests/suite/css.html b/tests/suite/css.html
index d490be5..af1c417 100644
--- a/tests/suite/css.html
+++ b/tests/suite/css.html
@@ -147,11 +147,19 @@
     body {
         font-family: Arial,
                      Helvetica;
+        a:link,
         a:hover,
         a:active {
             color: red;
         }
     }
+
+    /* Since DjCSS expects a space between the property and the value,
+       the following is currently broken */
+    body {
+        font-family:Arial,
+        Helvetica;
+    }
 </style>
 
 

From 86e32d0b678afdf7210f6f580060dd64259ecebe Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 20:33:04 +0100
Subject: [PATCH 15/45] Save the planet by not wasting resources to indent
 minified JavaScript

---
 djhtml/__main__.py | 4 ++++
 djhtml/modes.py    | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 6335db2..823ce76 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -60,6 +60,10 @@ def main():
         # Indent input file
         try:
             result = Mode(source).indent(options.tabwidth)
+        except modes.MaxLineLengthExceeded:
+            problematic_files += 1
+            _error(f"Maximum line length exceeded in {filename}")
+            continue
         except Exception:
             _error(
                 f"Fatal error while processing {filename}\n\n"
diff --git a/djhtml/modes.py b/djhtml/modes.py
index 5cbebb6..4a7ee77 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -10,6 +10,8 @@ class BaseMode:
 
     """
 
+    MAX_LINE_LENGTH = 10_000
+
     def __init__(self, source=None, return_mode=None):
         """
         Instantiate with source text before calling indent(), or
@@ -57,6 +59,9 @@ def tokenize(self):
         src = self.source
 
         while True:
+            if src.find("\n") > mode.MAX_LINE_LENGTH:
+                raise MaxLineLengthExceeded
+
             try:
                 # Split the source at the first occurrence of one of
                 # the current mode's raw tokens.
@@ -556,5 +561,9 @@ def create_token(self, raw_token, src, line):
         return token, mode
 
 
+class MaxLineLengthExceeded(Exception):
+    pass
+
+
 def compile_re(raw_tokens):
     return re.compile("(" + "|".join(raw_tokens) + ")")

From 07791e5cf29fcca7a19a5e3b6c4186c730379a25 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 20:35:26 +0100
Subject: [PATCH 16/45] Tabwidth guessing

---
 README.md          |  5 +++++
 djhtml/__main__.py | 18 +++++++++++++++++-
 djhtml/options.py  |  4 ++--
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index a6e4c7f..3bbb642 100644
--- a/README.md
+++ b/README.md
@@ -66,6 +66,11 @@ multi-line attribute values like this:
 </blockquote>
 ```
 
+## New! Tabwidth guessing
+
+Without the `-t` / `--tabwidth` argument, DjHTML no longer defaults to
+a tabwidth of 4 but instead guesses the correct tabwidth.
+
 
 ## New! Django middleware
 
diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 823ce76..6145e76 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -57,9 +57,25 @@ def main():
             _error(e)
             continue
 
+        # Guess tabwidth
+        if not options.tabwidth:
+            diff = 0
+            probabilities = [0] * 8
+            for line in source.split("\n"):
+                count = 0
+                for char in line:
+                    if char == " ":
+                        count += 1
+                    else:
+                        break
+                if 0 < abs(count - diff) < 8:
+                    probabilities[abs(count - diff)] += 1
+                diff = count
+            guess = probabilities.index(max(probabilities))
+
         # Indent input file
         try:
-            result = Mode(source).indent(options.tabwidth)
+            result = Mode(source).indent(options.tabwidth or guess or 4)
         except modes.MaxLineLengthExceeded:
             problematic_files += 1
             _error(f"Maximum line length exceeded in {filename}")
diff --git a/djhtml/options.py b/djhtml/options.py
index aaa48d1..a6d8e09 100644
--- a/djhtml/options.py
+++ b/djhtml/options.py
@@ -30,8 +30,8 @@
     "--tabwidth",
     metavar="N",
     type=int,
-    default=4,
-    help="tabwidth (default is 4)",
+    default=0,
+    help="tabwidth (default is to guess)",
 )
 parser.add_argument(
     "input_filenames",

From 528f78a3b894a8d17f11b4c5f75388523a0b31b3 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 21:09:44 +0100
Subject: [PATCH 17/45] Support the JavaScript "spread" operator

---
 djhtml/modes.py     |  7 ++++++-
 tests/suite/js.html | 20 +++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 4a7ee77..7927ad7 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -435,10 +435,15 @@ def create_token(self, raw_token, src, line):
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 r"\*/", mode=DjJS, return_mode=self
             )
+        elif raw_token.lstrip().startswith("..."):
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif not line and raw_token.lstrip().startswith("."):
             self.offsets["relative"] = 1
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
-        elif raw_token.lstrip().startswith(("case ", "default:")):
+        elif not line and raw_token.lstrip().startswith("case "):
+            token = Token.OpenDouble(raw_token, mode=DjJS)
+            return token, mode
+        elif not line and raw_token.lstrip() == "default:":
             token = Token.OpenDouble(raw_token, mode=DjJS)
             return token, mode
         elif raw_token in ["var ", "let ", "const "]:
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 0e560f0..dbfc5d1 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -159,9 +159,12 @@
             console.log('Rejected');
             break;
         }
-        default: {
-            console.log('Default');
-        }
+
+            // This one is broken because dictionary keys can also be
+            // called "default"...
+            default: {
+                console.log('Default');
+            }
     }
 </script>
 
@@ -335,3 +338,14 @@
     {% endif %}
     c = 3;
 </script>
+
+
+<!-- Things that look like case statements, and the "spread" operator -->
+<script>
+    var case = "foo"
+    var dict = {
+        case: "bar",
+        default: "baz",
+        ...params,
+    }
+</script>

From 1d71ec9ad5644a80aab0a8d5d384e966ce6517a8 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 21:41:13 +0100
Subject: [PATCH 18/45] JavaScript multi-line ternary operator

---
 djhtml/modes.py     | 2 +-
 tests/suite/js.html | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 7927ad7..e8a6a1c 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -437,7 +437,7 @@ def create_token(self, raw_token, src, line):
             )
         elif raw_token.lstrip().startswith("..."):
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
-        elif not line and raw_token.lstrip().startswith("."):
+        elif not line and raw_token.lstrip().startswith((".", ": ", "? ")):
             self.offsets["relative"] = 1
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif not line and raw_token.lstrip().startswith("case "):
diff --git a/tests/suite/js.html b/tests/suite/js.html
index dbfc5d1..df1cd7f 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -349,3 +349,11 @@
         ...params,
     }
 </script>
+
+
+<!-- Multi-line ternary operator -->
+<script>
+    foo
+        ? bar
+        : baz
+</script>

From e4951d8956f72672056ee8ad87b88e76b150dac2 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 22:16:43 +0100
Subject: [PATCH 19/45] Detect quoted strings with escaped quotes in DjJS mode

---
 djhtml/modes.py     | 8 ++++----
 tests/suite/js.html | 8 ++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index e8a6a1c..52e506b 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -386,12 +386,12 @@ class DjJS(DjTXT):
 
     RAW_TOKENS = DjTXT.RAW_TOKENS + [
         r"</script>",
-        r"\\.?",
+        r"\\.",
         r"//.*",
         r"/\*",
-        r'".*?"',
-        r"'.*?'",
-        r"`.*?`",
+        r'"(?:\\"|.)*?"',
+        r"'(?:\\'|.)*?'",
+        r"`(?:\\`|.)*?`",
         r"`",
         r"[\{\[\(\)\]\}]",
         r"var ",
diff --git a/tests/suite/js.html b/tests/suite/js.html
index df1cd7f..8c1d7e9 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -357,3 +357,11 @@
         ? bar
         : baz
 </script>
+
+
+<!-- Quoted strings with escaped quotes -->
+<script>
+    var foo = '\'{'
+    var bar = "\"{"
+    var baz = `\`{`
+</script>

From afb7e006504edf7f872b8256ca16ab9999c007ae Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sat, 18 Feb 2023 22:55:52 +0100
Subject: [PATCH 20/45] Improve JavaScript multi-line variable declaration

---
 djhtml/modes.py     | 4 ++--
 tests/suite/js.html | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 52e506b..99aa8b8 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -402,8 +402,8 @@ class DjJS(DjTXT):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.haskell = False
-        self.haskell_re = re.compile(r" *, [^ ]+ *=")
-        self.variable_re = re.compile(r" *[^ ]+ *=")
+        self.haskell_re = re.compile(r" *, ([\$\w]+ *=|[\$\w]+;?)")
+        self.variable_re = re.compile(r" *([\$\w]+ *=|[\$\w]+;?)")
         self.previous_line_ended_with_comma = False
 
     def create_token(self, raw_token, src, line):
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 8c1d7e9..41e851e 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -309,15 +309,15 @@
 <script>
     var a = 1,
         b = 2,
-        c = 3;
+        c;
     var foo = bar; let a = 1,
                        b = 2,
-                       c = 3
+                       c
 
     // Haskell-style indentation
     const a = 1
         , b = 2
-        , c = 3
+        , c
 
     // Notice the first line is the same
     const a = 1

From aa47f1ca4af480476b8c9113ffef4dd834d87473 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 15:02:08 +0100
Subject: [PATCH 21/45] Support single-line JavaScript if statements, under
 limited circumstances

---
 djhtml/modes.py     | 38 ++++++++++++++++----------------------
 tests/suite/js.html | 17 +++++++++++++++++
 2 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 99aa8b8..6134d69 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -28,9 +28,6 @@ def __init__(self, source=None, return_mode=None):
         # To keep track of the current offsets.
         self.offsets = dict(relative=0, absolute=0)
 
-        # To be used as a stack by some modes.
-        self.previous_offsets = []
-
     def indent(self, tabwidth):
         """
         Return the indented text as a single string.
@@ -232,19 +229,13 @@ def create_token(self, raw_token, src, line):
                     return_mode=self,
                 )
             elif self._has_closing_token(name, raw_token, src):
-                self.previous_offsets.append(self.offsets["relative"])
                 token = Token.Open(raw_token, mode=DjTXT, **self.offsets)
-                self.offsets["relative"] = 0
             elif name in self.CLOSING_AND_OPENING_TAGS:
                 token = Token.CloseAndOpen(raw_token, mode=DjTXT, **self.offsets)
             elif name.startswith("end"):
                 token = Token.Close(raw_token, mode=DjTXT, **self.offsets)
-                try:
-                    self.offsets["relative"] = self.previous_offsets.pop()
-                except IndexError:
-                    self.offsets["relative"] = 0
             else:
-                token = Token.Text(raw_token, mode=DjTXT, **self.offsets)
+                token = Token.Text(raw_token, mode=DjTXT)
         elif re.match(self.FMT_OFF, raw_token):
             token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
                 self.FMT_ON, mode=DjTXT, return_mode=self
@@ -397,10 +388,12 @@ class DjJS(DjTXT):
         r"var ",
         r"let ",
         r"const ",
+        r"if ",
     ]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.previous_offsets = []
         self.haskell = False
         self.haskell_re = re.compile(r" *, ([\$\w]+ *=|[\$\w]+;?)")
         self.variable_re = re.compile(r" *([\$\w]+ *=|[\$\w]+;?)")
@@ -418,15 +411,15 @@ def create_token(self, raw_token, src, line):
                 self.offsets["absolute"] = 0
 
         if raw_token in "{[(":
-            self.previous_offsets.append(self.offsets["absolute"])
-            self.offsets["absolute"] = 0
-            token = Token.Open(raw_token, mode=DjJS, **self.offsets)
+            self.previous_offsets.append(self.offsets.copy())
+            self.offsets = dict(relative=0, absolute=0)
+            token = Token.Open(raw_token, mode=DjJS)
         elif raw_token in ")]}":
-            token = Token.Close(raw_token, mode=DjJS, **self.offsets)
+            token = Token.Close(raw_token, mode=DjJS)
             try:
-                self.offsets["absolute"] = self.previous_offsets.pop()
+                self.offsets = self.previous_offsets.pop()
             except IndexError:
-                self.offsets["absolute"] = 0
+                self.offsets = dict(relative=0, absolute=0)
         elif raw_token == "`":
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 "`", mode=DjJS, return_mode=self
@@ -436,10 +429,12 @@ def create_token(self, raw_token, src, line):
                 r"\*/", mode=DjJS, return_mode=self
             )
         elif raw_token.lstrip().startswith("..."):
-            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
+            token = Token.Text(raw_token, mode=DjJS)
         elif not line and raw_token.lstrip().startswith((".", ": ", "? ")):
+            token = Token.Text(raw_token, mode=DjJS, relative=1)
+        elif not line and raw_token == "if ":
+            token = Token.Text(raw_token, mode=DjJS)
             self.offsets["relative"] = 1
-            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
         elif not line and raw_token.lstrip().startswith("case "):
             token = Token.OpenDouble(raw_token, mode=DjJS)
             return token, mode
@@ -448,7 +443,7 @@ def create_token(self, raw_token, src, line):
             return token, mode
         elif raw_token in ["var ", "let ", "const "]:
             self.haskell = False
-            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
+            token = Token.Text(raw_token, mode=DjJS)
             self.offsets["absolute"] = len(line) + len(raw_token)
         elif (
             not line
@@ -467,8 +462,8 @@ def create_token(self, raw_token, src, line):
         else:
             token, mode = super().create_token(raw_token, src, line)
 
-        # Reset relative offset after creating first token in line.
-        if not line:
+        # Reset relative offset in almost all cases
+        if raw_token != "if " and raw_token.strip() != ")":
             self.offsets["relative"] = 0
 
         if raw_token.rstrip().endswith(","):
@@ -516,7 +511,6 @@ def __init__(self, tagname, line, return_mode):
         self.offsets = dict(
             relative=-1 if line.indents else 0, absolute=len(line) + len(tagname) + 2
         )
-        self.previous_offsets = []
 
         # Pff...
         self.additional_offset = -len(tagname) - 1
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 41e851e..520f464 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -365,3 +365,20 @@
     var bar = "\"{"
     var baz = `\`{`
 </script>
+
+
+<!-- Single-line if statements -->
+<script>
+    if (foo)
+        bar
+    baz
+
+    if (
+        foo &&
+        bar
+    )
+        baz
+
+    if (foo) return
+    bar
+</script>

From 7d2d0762a503e0b5947d12775525d135af8faa1b Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 17:04:22 +0100
Subject: [PATCH 22/45] JavaScript multi-line variable assignment

---
 djhtml/modes.py     | 30 +++++++++++++++++++++---------
 tests/suite/js.html | 26 ++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 6134d69..5ecd843 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -401,25 +401,31 @@ def __init__(self, *args, **kwargs):
 
     def create_token(self, raw_token, src, line):
         mode = self
+        persist_relative_offset = False
 
         # Reset absolute offset in almost all cases
-        if not line and not self.haskell_re.match(raw_token):
-            if (
-                not self.variable_re.match(raw_token)
-                or not self.previous_line_ended_with_comma
-            ):
-                self.offsets["absolute"] = 0
+        if (
+            not line
+            and not self.haskell_re.match(raw_token)
+            and not (
+                self.variable_re.match(raw_token)
+                and self.previous_line_ended_with_comma
+            )
+        ):
+            self.offsets["absolute"] = 0
 
         if raw_token in "{[(":
+            token = Token.Open(raw_token, mode=DjJS, **self.offsets)
             self.previous_offsets.append(self.offsets.copy())
             self.offsets = dict(relative=0, absolute=0)
-            token = Token.Open(raw_token, mode=DjJS)
         elif raw_token in ")]}":
             token = Token.Close(raw_token, mode=DjJS)
             try:
                 self.offsets = self.previous_offsets.pop()
             except IndexError:
                 self.offsets = dict(relative=0, absolute=0)
+            if raw_token == ")":
+                persist_relative_offset = True
         elif raw_token == "`":
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 "`", mode=DjJS, return_mode=self
@@ -433,14 +439,20 @@ def create_token(self, raw_token, src, line):
         elif not line and raw_token.lstrip().startswith((".", ": ", "? ")):
             token = Token.Text(raw_token, mode=DjJS, relative=1)
         elif not line and raw_token == "if ":
-            token = Token.Text(raw_token, mode=DjJS)
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
             self.offsets["relative"] = 1
+            persist_relative_offset = True
         elif not line and raw_token.lstrip().startswith("case "):
             token = Token.OpenDouble(raw_token, mode=DjJS)
             return token, mode
         elif not line and raw_token.lstrip() == "default:":
             token = Token.OpenDouble(raw_token, mode=DjJS)
             return token, mode
+        elif raw_token.rstrip().endswith(("=", ":")):
+            token = Token.Text(raw_token, mode=DjJS)
+            if not line.indents:
+                self.offsets["relative"] = 1
+                persist_relative_offset = True
         elif raw_token in ["var ", "let ", "const "]:
             self.haskell = False
             token = Token.Text(raw_token, mode=DjJS)
@@ -463,7 +475,7 @@ def create_token(self, raw_token, src, line):
             token, mode = super().create_token(raw_token, src, line)
 
         # Reset relative offset in almost all cases
-        if raw_token != "if " and raw_token.strip() != ")":
+        if not persist_relative_offset and raw_token.strip():
             self.offsets["relative"] = 0
 
         if raw_token.rstrip().endswith(","):
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 520f464..c197cd8 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -382,3 +382,29 @@
     if (foo) return
     bar
 </script>
+
+
+<!-- Multi-line variables and dictionaries -->
+<script>
+    var foo =
+        bar
+
+    const func =
+        function() {
+            return
+        })
+
+    let dict =
+        {
+            foo:
+                bar
+        }
+
+    // This one is broken
+    let dict = {
+        default:
+            foo
+            bar:
+                baz
+    }
+</script>

From be2dde0f464a2a6748bfa3fc8c65c084bd710a95 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 17:13:33 +0100
Subject: [PATCH 23/45] Count tabs as 4 spaces

---
 djhtml/__main__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 6145e76..241ff53 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -66,6 +66,8 @@ def main():
                 for char in line:
                     if char == " ":
                         count += 1
+                    elif char == "\t":
+                        count += 4
                     else:
                         break
                 if 0 < abs(count - diff) < 8:

From df3f4930e0c166c62615381776abdd79177607c5 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 22:35:15 +0100
Subject: [PATCH 24/45] Single-line JavaScript else, for and while statements

---
 djhtml/modes.py     | 59 ++++++++++++++++++++++++++++-----------------
 tests/suite/js.html | 59 +++++++++++++++++++++++++++++----------------
 2 files changed, 75 insertions(+), 43 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 5ecd843..68b2ead 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -376,27 +376,33 @@ class DjJS(DjTXT):
     """
 
     RAW_TOKENS = DjTXT.RAW_TOKENS + [
-        r"</script>",
         r"\\.",
         r"//.*",
         r"/\*",
-        r'"(?:\\"|.)*?"',
-        r"'(?:\\'|.)*?'",
-        r"`(?:\\`|.)*?`",
+        r"[\$\w-]+:(?= *\n)",  # label:
+        r'"(?:\\"|.)*?":(?= *\n)',  # "label":
+        r"'(?:\\'|.)*?':(?= *\n)",  # 'label':
+        r'"(?:\\"|.)*?"',  # "string"
+        r"'(?:\\'|.)*?'",  # 'string'
+        r"`(?:\\`|.)*?`",  # `string`
         r"`",
         r"[\{\[\(\)\]\}]",
         r"var ",
         r"let ",
         r"const ",
-        r"if ",
+        r"if(?= *\()",
+        r"else(?= *\n)",
+        r"for(?= *\()",
+        r"while(?= *\()",
+        r"</script>",
     ]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.previous_offsets = []
         self.haskell = False
-        self.haskell_re = re.compile(r" *, ([\$\w]+ *=|[\$\w]+;?)")
-        self.variable_re = re.compile(r" *([\$\w]+ *=|[\$\w]+;?)")
+        self.haskell_re = re.compile(r" *, ([\$\w-]+ *=|[\$\w-]+;?)")
+        self.variable_re = re.compile(r" *([\$\w-]+ *=|[\$\w-]+;?)")
         self.previous_line_ended_with_comma = False
 
     def create_token(self, raw_token, src, line):
@@ -414,18 +420,17 @@ def create_token(self, raw_token, src, line):
         ):
             self.offsets["absolute"] = 0
 
+        # Opening and closing tokens
         if raw_token in "{[(":
-            token = Token.Open(raw_token, mode=DjJS, **self.offsets)
             self.previous_offsets.append(self.offsets.copy())
             self.offsets = dict(relative=0, absolute=0)
+            token = Token.Open(raw_token, mode=DjJS)
         elif raw_token in ")]}":
-            token = Token.Close(raw_token, mode=DjJS)
-            try:
+            if self.previous_offsets:
                 self.offsets = self.previous_offsets.pop()
-            except IndexError:
-                self.offsets = dict(relative=0, absolute=0)
             if raw_token == ")":
                 persist_relative_offset = True
+            token = Token.Close(raw_token, mode=DjJS)
         elif raw_token == "`":
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 "`", mode=DjJS, return_mode=self
@@ -434,27 +439,34 @@ def create_token(self, raw_token, src, line):
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 r"\*/", mode=DjJS, return_mode=self
             )
+        elif raw_token.startswith("//"):
+            token = Token.Text(raw_token, mode=DjJS, ignore=True)
+
+        # "Double" tokens
+        elif not line and raw_token.lstrip().startswith("case "):
+            token = Token.OpenDouble(raw_token, mode=DjJS)
+            return token, mode
+        elif raw_token.rstrip().endswith("default:"):
+            token = Token.OpenDouble(raw_token, mode=DjJS)
+            return token, mode
+
+        # Tokens that mess with relative offset
         elif raw_token.lstrip().startswith("..."):
             token = Token.Text(raw_token, mode=DjJS)
         elif not line and raw_token.lstrip().startswith((".", ": ", "? ")):
             token = Token.Text(raw_token, mode=DjJS, relative=1)
-        elif not line and raw_token == "if ":
+        elif not line and raw_token in ["if", "else", "for", "while"]:
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
-            self.offsets["relative"] = 1
+            self.offsets["relative"] += 1
             persist_relative_offset = True
-        elif not line and raw_token.lstrip().startswith("case "):
-            token = Token.OpenDouble(raw_token, mode=DjJS)
-            return token, mode
-        elif not line and raw_token.lstrip() == "default:":
-            token = Token.OpenDouble(raw_token, mode=DjJS)
-            return token, mode
         elif raw_token.rstrip().endswith(("=", ":")):
-            token = Token.Text(raw_token, mode=DjJS)
+            token = Token.Text(raw_token, mode=DjJS, **self.offsets)
             if not line.indents:
                 self.offsets["relative"] = 1
                 persist_relative_offset = True
+
+        # Tokens that mess with absolute offset
         elif raw_token in ["var ", "let ", "const "]:
-            self.haskell = False
             token = Token.Text(raw_token, mode=DjJS)
             self.offsets["absolute"] = len(line) + len(raw_token)
         elif (
@@ -466,6 +478,8 @@ def create_token(self, raw_token, src, line):
             self.haskell = True
             self.offsets["absolute"] -= 2
             token = Token.Text(raw_token, mode=DjJS, **self.offsets)
+
+        # Get out of this mess!
         elif raw_token == "</script>":
             token, mode = (
                 Token.Close(raw_token, mode=self.return_mode.__class__),
@@ -478,6 +492,7 @@ def create_token(self, raw_token, src, line):
         if not persist_relative_offset and raw_token.strip():
             self.offsets["relative"] = 0
 
+        # Remember whether the line ended with a comma
         if raw_token.rstrip().endswith(","):
             self.previous_line_ended_with_comma = True
         else:
diff --git a/tests/suite/js.html b/tests/suite/js.html
index c197cd8..bbac488 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -157,14 +157,12 @@
         }
         case 'rejected': {
             console.log('Rejected');
+        }
+            break;
+        default: {
+            console.log('Default');
             break;
         }
-
-            // This one is broken because dictionary keys can also be
-            // called "default"...
-            default: {
-                console.log('Default');
-            }
     }
 </script>
 
@@ -331,6 +329,11 @@
           func2 = function() { return 2 },
           func3 = function() { return 3 }
 
+    const foo = bar
+          baz = {
+        ...
+    }
+
     // We are not supporting madness like this
     const a = 1,
     {% if %}
@@ -345,7 +348,7 @@
     var case = "foo"
     var dict = {
         case: "bar",
-        default: "baz",
+        "default:": "baz",
         ...params,
     }
 </script>
@@ -367,20 +370,40 @@
 </script>
 
 
-<!-- Single-line if statements -->
+<!-- Single-line statements -->
 <script>
     if (foo)
         bar
-    baz
+
+    if (foo) return
+    bar
 
     if (
         foo &&
         bar
     )
         baz
+    else
+        return
 
-    if (foo) return
-    bar
+    if (foo)
+        if (bar)
+            if (baz)
+                return
+    else
+        console.log("Footgun detected!")
+
+    leitmotif ()
+    elsewhere ();
+    meanwhile (true)
+
+    while (true)
+        break
+
+    for (i=1, 1<10, i++)
+    {
+        break
+    }
 </script>
 
 
@@ -394,17 +417,11 @@
             return
         })
 
-    let dict =
-        {
-            foo:
-                bar
-        }
-
-    // This one is broken
+    // Note that unquoted "default" is a reserved keyword
     let dict = {
-        default:
+        "default":
             foo
-            bar:
-                baz
+        bar:
+            baz
     }
 </script>

From 2443375bed10f837643bd9574cee77594a6a2f2e Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 22:36:22 +0100
Subject: [PATCH 25/45] Skip empty lines when guessing tabwidth

---
 djhtml/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 241ff53..711919e 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -70,7 +70,7 @@ def main():
                         count += 4
                     else:
                         break
-                if 0 < abs(count - diff) < 8:
+                if count and 0 < abs(count - diff) < 8:
                     probabilities[abs(count - diff)] += 1
                 diff = count
             guess = probabilities.index(max(probabilities))

From 225de110fb5896bda6658983b4327b9bea0834d8 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 22:45:34 +0100
Subject: [PATCH 26/45] Limit guesses to 2, 4, or 8

---
 djhtml/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 711919e..a9bad05 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -70,7 +70,7 @@ def main():
                         count += 4
                     else:
                         break
-                if count and 0 < abs(count - diff) < 8:
+                if count and abs(count - diff) in [2, 4, 8]:
                     probabilities[abs(count - diff)] += 1
                 diff = count
             guess = probabilities.index(max(probabilities))

From 2bb321b369af6249c048989897a640e276772d0f Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 22:48:04 +0100
Subject: [PATCH 27/45] Fix off-by-one error

---
 djhtml/__main__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index a9bad05..a2190f2 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -60,7 +60,7 @@ def main():
         # Guess tabwidth
         if not options.tabwidth:
             diff = 0
-            probabilities = [0] * 8
+            probabilities = [0] * 9
             for line in source.split("\n"):
                 count = 0
                 for char in line:

From aaa328dd6ddb5335740ab28d2b28fc956751c1dc Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 23:18:06 +0100
Subject: [PATCH 28/45] Fix false negative unittest

---
 djhtml/modes.py     | 4 ++--
 tests/suite/js.html | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 68b2ead..e971018 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -401,8 +401,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.previous_offsets = []
         self.haskell = False
-        self.haskell_re = re.compile(r" *, ([\$\w-]+ *=|[\$\w-]+;?)")
-        self.variable_re = re.compile(r" *([\$\w-]+ *=|[\$\w-]+;?)")
+        self.haskell_re = re.compile(r"^ *, ([\$\w-]+ *=|[\$\w-]+;?)")
+        self.variable_re = re.compile(r"^ *([\$\w-]+ *=|[\$\w-]+;?)")
         self.previous_line_ended_with_comma = False
 
     def create_token(self, raw_token, src, line):
diff --git a/tests/suite/js.html b/tests/suite/js.html
index bbac488..66c054d 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -329,7 +329,7 @@
           func2 = function() { return 2 },
           func3 = function() { return 3 }
 
-    const foo = bar
+    const foo = bar,
           baz = {
         ...
     }

From 98dccf64d894cd03746d3fe533c8eb83e2e18324 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 23:30:50 +0100
Subject: [PATCH 29/45] Fix template tags in multi-line HTML elements

---
 djhtml/modes.py       |  2 +-
 tests/suite/html.html | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index e971018..a17c180 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -235,7 +235,7 @@ def create_token(self, raw_token, src, line):
             elif name.startswith("end"):
                 token = Token.Close(raw_token, mode=DjTXT, **self.offsets)
             else:
-                token = Token.Text(raw_token, mode=DjTXT)
+                token = Token.Text(raw_token, mode=DjTXT, **self.offsets)
         elif re.match(self.FMT_OFF, raw_token):
             token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
                 self.FMT_ON, mode=DjTXT, return_mode=self
diff --git a/tests/suite/html.html b/tests/suite/html.html
index a47ab81..dbfa6a9 100644
--- a/tests/suite/html.html
+++ b/tests/suite/html.html
@@ -320,6 +320,18 @@ <h1>Title</h1>
                      id="4"/></div>
 
 
+<!-- Multi-line elements with template tags -->
+<a href=""
+   {% include "attrs.html" %}
+   class="
+          {% block classes %}
+              foo
+              bar
+          {% endblock %}
+         "
+/>
+
+
 <!-- The example from the README -->
 <blockquote cite="John Lennon"
             style="color: dimgray;

From dc16645411d27a2bad7f8ce3bd3513f0d0d9ec05 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Sun, 19 Feb 2023 23:55:07 +0100
Subject: [PATCH 30/45] Allow single-quoted HTML attribute values

---
 djhtml/modes.py       | 13 +++++++------
 tests/suite/html.html | 10 ++++++++++
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index a17c180..4db0b9e 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -528,7 +528,7 @@ class InsideHTMLTag(DjTXT):
 
     """
 
-    RAW_TOKENS = DjTXT.RAW_TOKENS + [r"/?>", r"[^ ='\">/]+=", r'"']
+    RAW_TOKENS = DjTXT.RAW_TOKENS + [r"/?>", r"[^ ='\">/]+=", r'"', r"'"]
 
     def __init__(self, tagname, line, return_mode):
         self.tagname = tagname
@@ -552,14 +552,15 @@ def create_token(self, raw_token, src, line):
         if "text/template" in raw_token:
             self.tagname = ""
 
-        if raw_token == '"':
+        if raw_token in ['"', "'"]:
             if self.inside_attr:
-                self.inside_attr = False
-                self.offsets["absolute"] -= 1
                 token = Token.Text(raw_token, mode=InsideHTMLTag, **self.offsets)
-                self.offsets["absolute"] = self.previous_offset
+                if self.inside_attr == raw_token:
+                    self.inside_attr = False
+                    token.absolute = self.offsets["absolute"] - 1
+                    self.offsets["absolute"] = self.previous_offset
             else:
-                self.inside_attr = True
+                self.inside_attr = raw_token
                 self.previous_offset = self.offsets["absolute"]
                 self.offsets["absolute"] += self.additional_offset
                 token = Token.Text(raw_token, mode=InsideHTMLTag, **self.offsets)
diff --git a/tests/suite/html.html b/tests/suite/html.html
index dbfa6a9..55bca14 100644
--- a/tests/suite/html.html
+++ b/tests/suite/html.html
@@ -340,3 +340,13 @@ <h1>Title</h1>
                   ">
     It's weird not to be weird.
 </blockquote>
+
+
+<!-- Who knew HTML allowed single quotes? -->
+<blockquote class='foo
+                  '
+            class='bar"
+                  '
+            class="baz'
+                  "
+/>

From f6087f311bc9e256aa7f67b57aa07f33db3c752c Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Mon, 20 Feb 2023 00:08:40 +0100
Subject: [PATCH 31/45] Less regexes

---
 djhtml/modes.py     |  4 +---
 tests/suite/js.html | 11 +++++++++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 4db0b9e..c03c1cb 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -379,9 +379,7 @@ class DjJS(DjTXT):
         r"\\.",
         r"//.*",
         r"/\*",
-        r"[\$\w-]+:(?= *\n)",  # label:
-        r'"(?:\\"|.)*?":(?= *\n)',  # "label":
-        r"'(?:\\'|.)*?':(?= *\n)",  # 'label':
+        r"[\$\w-]+:",
         r'"(?:\\"|.)*?"',  # "string"
         r"'(?:\\'|.)*?'",  # 'string'
         r"`(?:\\`|.)*?`",  # `string`
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 66c054d..0f29cb3 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -217,6 +217,15 @@
 </script>
 
 
+<!-- Succinct switch statement -->
+<script>
+    switch (foo) {
+        case bar: break
+        default: return
+    }
+</script>
+
+
 <!-- Things that look like switch, but aren't -->
 <script>
     function switchBoo () {
@@ -247,6 +256,7 @@
     }
 </script>
 
+
 <!-- More function chaining -->
 <script>
     window.fetch('/test.html')
@@ -277,6 +287,7 @@
     </script>
 </head>
 
+
 <!-- Too much closing tokens -->
 <head>
     <script>

From 60f9c110641356fbb7f81009bfa8236f5c1dd1af Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Mon, 20 Feb 2023 19:21:18 +0100
Subject: [PATCH 32/45] Improve guessing algorithm (thanks SJ!)

---
 djhtml/__main__.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index a2190f2..1d43070 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -59,20 +59,14 @@ def main():
 
         # Guess tabwidth
         if not options.tabwidth:
-            diff = 0
+            prev = 0
             probabilities = [0] * 9
-            for line in source.split("\n"):
-                count = 0
-                for char in line:
-                    if char == " ":
-                        count += 1
-                    elif char == "\t":
-                        count += 4
-                    else:
-                        break
-                if count and abs(count - diff) in [2, 4, 8]:
-                    probabilities[abs(count - diff)] += 1
-                diff = count
+            for line in source.splitlines():
+                if line and not line.isspace():
+                    depth = _get_depth(line)
+                    if abs(depth - prev) in [2, 4, 8]:
+                        probabilities[abs(depth - prev)] += 1
+                    prev = depth
             guess = probabilities.index(max(probabilities))
 
         # Indent input file
@@ -168,6 +162,18 @@ def _verify_changed(source, result):
     return changed
 
 
+def _get_depth(line):
+    count = 0
+    for char in line:
+        if char == " ":
+            count += 1
+        elif char == "\t":
+            count += 4
+        else:
+            break
+    return count
+
+
 def _info(msg):
     if not options.quiet:
         print(msg, file=sys.stderr)

From 8c6af6dad5e6df5c23175efb89ed65e78a482464 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Mon, 20 Feb 2023 19:41:10 +0100
Subject: [PATCH 33/45] Remove existing indentation from test suite before
 testing

---
 tests/test_suite.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_suite.py b/tests/test_suite.py
index f56054e..d1979e2 100644
--- a/tests/test_suite.py
+++ b/tests/test_suite.py
@@ -21,5 +21,11 @@ def test_available_files(self):
     def _test_file(self, filename):
         with open(os.path.join(self.DIR, filename)) as f:
             expected_output = f.read()
-        actual_output = DjHTML(expected_output).indent(4)
+
+        # Indent the expected output to 0 (no indentation)
+        unindented = DjHTML(expected_output).indent(0)
+        self.assertNotEqual(unindented, expected_output)
+
+        # Re-indent the unindented output to 4
+        actual_output = DjHTML(unindented).indent(4)
         self.assertEqual(expected_output, actual_output)

From 5d1452366eed7637c6d478c867e884adc71d43e0 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Mon, 20 Feb 2023 22:39:35 +0100
Subject: [PATCH 34/45] Omit empty lines from middleware

---
 djhtml/middleware.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/djhtml/middleware.py b/djhtml/middleware.py
index 0a04e5d..a1e46e3 100644
--- a/djhtml/middleware.py
+++ b/djhtml/middleware.py
@@ -29,7 +29,15 @@ def __call__(self, request):
                     "Please load DjHTMLMiddleware _after_ CommonMiddleware (otherwise"
                     ' the "Content-Length" header will be incorrect)'
                 )
-            response.content = (
-                DjHTML(response.content.decode()).indent(TABWIDTH).encode()
-            )
+
+            response.content = "\n".join(
+                [
+                    line
+                    for line in DjHTML(response.content.decode())
+                    .indent(TABWIDTH)
+                    .splitlines()
+                    if line
+                ]
+            ).encode()
+
         return response

From e73ff43a1e1d37aedfdbe3fae2a86cc9e40c6b03 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 00:07:56 +0100
Subject: [PATCH 35/45] Cleanup help message and remove `--quiet` option

All status messages are written to stderr, so they can easily be
redirected to /dev/null when desired.
---
 README.md          |  2 +-
 djhtml/__main__.py |  6 ++----
 djhtml/options.py  | 47 ++++++++++++++++++++++++++++++++++------------
 setup.cfg          |  2 ++
 4 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 3bbb642..8926fdb 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 ***A pure-Python Django/Jinja template indenter without dependencies.***
 
-DjHTML indents mixed HTML/CSS/Javascript templates that contain
+DjHTML indents mixed HTML/CSS/JavaScript templates that contain
 [Django](https://docs.djangoproject.com/en/stable/ref/templates/language/)
 or [Jinja](https://jinja.palletsprojects.com/templates/) template
 tags. It works similar to other code-formatting tools such as
diff --git a/djhtml/__main__.py b/djhtml/__main__.py
index 1d43070..7e062e8 100644
--- a/djhtml/__main__.py
+++ b/djhtml/__main__.py
@@ -93,8 +93,7 @@ def main():
         # Write output file
         if not options.check:
             if filename == "-":
-                if not options.quiet:
-                    print(result, end="")
+                print(result, end="")
             elif changed:
                 try:
                     with open(filename, "w") as output_file:
@@ -175,8 +174,7 @@ def _get_depth(line):
 
 
 def _info(msg):
-    if not options.quiet:
-        print(msg, file=sys.stderr)
+    print(msg, file=sys.stderr)
 
 
 def _error(msg):
diff --git a/djhtml/options.py b/djhtml/options.py
index a6d8e09..3d05b85 100644
--- a/djhtml/options.py
+++ b/djhtml/options.py
@@ -8,15 +8,33 @@
 
 import argparse
 import sys
+from importlib.metadata import version
 
 parser = argparse.ArgumentParser(
     description=(
-        "DjHTML is a fully automatic template indenter that works with mixed"
-        " HTML/CSS/Javascript templates that contain Django or Jinja template"
-        " tags. It works similar to other code-formatting tools such as Black and"
-        " interoperates nicely with pre-commit. Full documentation can be found at"
-        " https://github.com/rtts/djhtml"
+        """
+DjHTML indents mixed HTML/CSS/JavaScript templates that
+contain Django or Jinja template tags. It works similar to
+other code-formatting tools such as Black and interoperates
+nicely with pre-commit."""
     ),
+    epilog="Full documentation at https://github.com/rtts/djhtml",
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    add_help=False,
+)
+parser.add_argument(
+    "-h",
+    "--help",
+    dest="show_help",
+    action="store_true",
+    help="show this help message and exit",
+)
+parser.add_argument(
+    "-v",
+    "--version",
+    dest="show_version",
+    action="store_true",
+    help="show version number and exit",
 )
 parser.add_argument(
     "-c",
@@ -24,29 +42,34 @@
     action="store_true",
     help="check indentation without modifying files",
 )
-parser.add_argument("-q", "--quiet", action="store_true", help="be quiet")
 parser.add_argument(
     "-t",
     "--tabwidth",
     metavar="N",
     type=int,
     default=0,
-    help="tabwidth (default is to guess)",
+    help="tabwidth (the default is to guess)",
 )
 parser.add_argument(
     "input_filenames",
-    metavar="filename",
-    nargs="+",
-    help="input filenames (either paths or directories)",
+    metavar="SOURCE",
+    nargs="*",
+    help="file or directory name(s) to indent",
 )
-parser.add_argument("-d", "--debug", action="store_true", help="debug mode")
+parser.add_argument("-d", "--debug", action="store_true", help=argparse.SUPPRESS)
 parser.add_argument("-i", "--in-place", action="store_true", help=argparse.SUPPRESS)
 
 # Parse arguments and assign attributes to self
 self = sys.modules[__name__]
 args = parser.parse_args(namespace=self)
 
-if self.in_place:
+if show_version:
+    print(version("djhtml"))
+    sys.exit()
+elif show_help or not input_filenames:
+    parser.print_help()
+    sys.exit()
+elif in_place:
     sys.exit(
         """
 You have called DjHTML with the -i or --in-place argument which
diff --git a/setup.cfg b/setup.cfg
index 8a2419c..fb2d9c2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -40,6 +40,8 @@ console_scripts =
 [flake8]
 max-line-length = 88
 extend-ignore = E203
+per-file-ignores =
+    djhtml/options.py: F821
 
 [isort]
 profile = black

From 9f37df8efe6fb9ea20f2d6b49d6527cffd26104b Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 01:18:03 +0100
Subject: [PATCH 36/45] Update README with new examples

---
 README.md             | 109 ++++++++++++++++++++----------------------
 tests/suite/css.html  |  13 +++++
 tests/suite/html.html |  11 +++--
 3 files changed, 72 insertions(+), 61 deletions(-)

diff --git a/README.md b/README.md
index 8926fdb..65c041e 100644
--- a/README.md
+++ b/README.md
@@ -14,74 +14,73 @@ whitespace at the beginning of lines. It will not insert newlines or
 other characters. The goal is to correctly indent already
 well-structured templates, not to fix broken ones.
 
-For example, consider the following incorrectly indented template:
 
-```jinja
-<!doctype html>
-<html>
-    <body>
-        {% block content %}
-        Hello, world!
-        {% endblock %}
-        <script>
-            $(function() {
-            console.log('Hi mom!');
-            });
-        </script>
-    </body>
-</html>
-```
+### New! Multi-line HTML elements
 
-This is what it will look like after processing by DjHTML:
+As of version 3, DjHTML indents multi-line HTML elements and
+multi-line attribute values like this:
 
 ```jinja
-<!doctype html>
-<html>
-    <body>
-        {% block content %}
-            Hello, world!
-        {% endblock %}
-        <script>
-            $(function() {
-                console.log('Hi mom!');
-            });
-        </script>
-    </body>
-</html>
+<blockquote cite="Guido Van Rossum"
+            style="font-style: italic;
+                   {% if dark_mode %}
+                       background: black;
+                   {% endif %}
+                  ">
+    Don't you hate code that's not properly indented?
+</blockquote>
 ```
 
 
-## New! Multi-line HTML elements
+### New! Multi-line CSS indentation
 
-As of version 3, DjHTML indents multi-line HTML elements and
-multi-line attribute values like this:
+Multi-line CSS values are now continued at the same indentation level:
 
-```html
-<blockquote cite="John Lennon"
-            style="color: dimgray;
-                   font-style: italic;
-                   border-left: 5px solid gray;
-                  ">
-    It's weird not to be weird.
-</blockquote>
+```jinja
+<style>
+    @font-face {
+        font-family: Helvetica;
+        src: {% for format, filename in licensed_fonts %}
+                 url('{% static filename %}') format('{{ format }}'),
+             {% endfor %}
+             url('Arial.woff2') format('woff2'),
+             url('Arial.woff') format('woff');
+    }
+</style>
 ```
 
-## New! Tabwidth guessing
 
-Without the `-t` / `--tabwidth` argument, DjHTML no longer defaults to
-a tabwidth of 4 but instead guesses the correct tabwidth.
+### New! Improved JavaScript indentation
+
+Many new JavaScript indention rules have been added, such as the
+indentation of method chaining:
+
+```jinja
+<script>
+    window.fetch('/test.html')
+        .then((html) => {
+            document.body.innerHTML = html;
+            {% block extra_statements %}
+            {% endblock %}
+        });
+</script>
+```
 
 
-## New! Django middleware
+### New! Django middleware
 
 To automatically indent all the HTML responses from your Django web
 application, add the following to your settings file:
 
-    TABWIDTH = 4
-    MIDDLEWARE += ['djhtml.middleware.DjHTMLMiddleware']
+```python
+MIDDLEWARE += ['djhtml.middleware.DjHTMLMiddleware']
+```
+
+
+### New! Tabwidth guessing
 
-(Caution: when used in production, it is advised to use some kind of
-[caching](https://docs.djangoproject.com/en/stable/topics/cache/).)
+Without the `-t` / `--tabwidth` argument, DjHTML no longer defaults to
+a tabwidth of 4 but instead guesses the correct tabwidth.
 
 
 ## Installation
@@ -127,13 +126,11 @@ You can exclude specific lines from being processed with the
 `{# fmt:off #}` and `{# fmt:on #}` operators:
 
 ```jinja
-<div class="
-    {# fmt:off #}
-      ,-._|\
-     /     .\
-     \_,--._/
-    {# fmt:on #}
-           "/>
+{# fmt:off #}
+   ,-._|\
+  /     .\
+  \_,--._/
+{# fmt:on #}
 ```
 
 Contents inside `<pre> ... </pre>`, `<!-- ... --->`, `/* ... */`, and
diff --git a/tests/suite/css.html b/tests/suite/css.html
index af1c417..477eb95 100644
--- a/tests/suite/css.html
+++ b/tests/suite/css.html
@@ -53,6 +53,19 @@
 </style>
 
 
+<!-- The example from the README -->
+<style>
+    @font-face {
+        font-family: Helvetica;
+        src: {% for format, filename in licensed_fonts %}
+                 url('{% static filename %}') format('{{ format }}'),
+             {% endfor %}
+             url('Arial.woff2') format('woff2'),
+             url('Arial.woff') format('woff');
+    }
+</style>
+
+
 <!-- @media -->
 <style>
     @media(max-width: 800px) {
diff --git a/tests/suite/html.html b/tests/suite/html.html
index 55bca14..bc143dc 100644
--- a/tests/suite/html.html
+++ b/tests/suite/html.html
@@ -333,12 +333,13 @@ <h1>Title</h1>
 
 
 <!-- The example from the README -->
-<blockquote cite="John Lennon"
-            style="color: dimgray;
-                   font-style: italic;
-                   border-left: 5px solid gray;
+<blockquote cite="Guido Van Rossum"
+            style="font-style: italic;
+                   {% if dark_mode %}
+                       background: black;
+                   {% endif %}
                   ">
-    It's weird not to be weird.
+    Don't you hate code that's not properly indented?
 </blockquote>
 
 

From fe4dde362a1d19772bee3e9ac4f5d80189ef93b5 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 10:49:23 +0100
Subject: [PATCH 37/45] Remove Django middleware
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

While it was a nice gimmick, actually using this in production would be a
Bad Idea™
---
 README.md            | 10 ----------
 djhtml/middleware.py | 43 -------------------------------------------
 2 files changed, 53 deletions(-)
 delete mode 100644 djhtml/middleware.py

diff --git a/README.md b/README.md
index 65c041e..1e5f84c 100644
--- a/README.md
+++ b/README.md
@@ -67,16 +67,6 @@ indentation of method chaining:
 ```
 
 
-### New! Django middleware
-
-To automatically indent all the HTML responses from your Django web
-application, add the following to your settings file:
-
-```python
-MIDDLEWARE += ['djhtml.middleware.DjHTMLMiddleware']
-```
-
-
 ### New! Tabwidth guessing
 
 Without the `-t` / `--tabwidth` argument, DjHTML no longer defaults to
diff --git a/djhtml/middleware.py b/djhtml/middleware.py
deleted file mode 100644
index a1e46e3..0000000
--- a/djhtml/middleware.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
-
-from djhtml.modes import DjHTML
-
-try:
-    TABWIDTH = settings.TABWIDTH
-except AttributeError:
-    TABWIDTH = 4
-
-
-class DjHTMLMiddleware:
-    """
-    Django middleware class to indent HTML responses.
-
-    """
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        response = self.get_response(request)
-        if (
-            "Content-Type" in response
-            and response["Content-Type"] == "text/html; charset=utf-8"
-        ):
-            if "Content-Length" in response:
-                raise ImproperlyConfigured(
-                    "Please load DjHTMLMiddleware _after_ CommonMiddleware (otherwise"
-                    ' the "Content-Length" header will be incorrect)'
-                )
-
-            response.content = "\n".join(
-                [
-                    line
-                    for line in DjHTML(response.content.decode())
-                    .indent(TABWIDTH)
-                    .splitlines()
-                    if line
-                ]
-            ).encode()
-
-        return response

From 09b419ad5ba2c1355688e896f5bc0033bd9bed89 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 16:53:52 +0100
Subject: [PATCH 38/45] Update regular expression to match quoted strings [wip]

---
 djhtml/modes.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index c03c1cb..7e282ce 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -380,9 +380,9 @@ class DjJS(DjTXT):
         r"//.*",
         r"/\*",
         r"[\$\w-]+:",
-        r'"(?:\\"|.)*?"',  # "string"
-        r"'(?:\\'|.)*?'",  # 'string'
-        r"`(?:\\`|.)*?`",  # `string`
+        r'"(?:\\.|[^\\])*"',  # "string"
+        r"'(?:\\.|[^\\])*'",  # 'string'
+        r"`(?:\\.|[^\\])*`",  # `string`
         r"`",
         r"[\{\[\(\)\]\}]",
         r"var ",

From 92d68aed601d600f20cd87fe6d6664c0f2c31a02 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 17:45:37 +0100
Subject: [PATCH 39/45] Fix regular expression to match quoted strings

---
 djhtml/modes.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 7e282ce..68cff00 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -380,9 +380,9 @@ class DjJS(DjTXT):
         r"//.*",
         r"/\*",
         r"[\$\w-]+:",
-        r'"(?:\\.|[^\\])*"',  # "string"
-        r"'(?:\\.|[^\\])*'",  # 'string'
-        r"`(?:\\.|[^\\])*`",  # `string`
+        r'"(?:\\.|[^\\"])*"',  # "string"
+        r"'(?:\\.|[^\\'])*'",  # 'string'
+        r"`(?:\\.|[^\\`])*`",  # `string`
         r"`",
         r"[\{\[\(\)\]\}]",
         r"var ",

From 1c56e7957f3f50352991829bc1b0f40219ba5767 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 17:51:18 +0100
Subject: [PATCH 40/45] Remove superfluous backslashes

---
 djhtml/modes.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 68cff00..d837fc0 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -187,7 +187,7 @@ class DjTXT(BaseMode):
 
     RAW_TOKENS = [
         r"\n",
-        r"{%[-\+]?.*?[-\+]?%}",
+        r"{%[-+]?.*?[-+]?%}",
         r"{#.*?#}",
         r"{#",
         r"{{.*?}}",
@@ -211,7 +211,7 @@ class DjTXT(BaseMode):
     }
     FMT_ON = r"{# fmt:on #}"
     FMT_OFF = r"{# fmt:off #}"
-    OPENING_TAG = r"{%[-\+]? *(\w+).*?[-\+]?%}"
+    OPENING_TAG = r"{%[-+]? *(\w+).*?[-+]?%}"
 
     def create_token(self, raw_token, src, line):
         mode = self
@@ -224,7 +224,7 @@ def create_token(self, raw_token, src, line):
                     mode=DjTXT,
                     ignore=True,
                 ), Comment(
-                    r"{%[-\+]? *end" + name + r"(?: .*?|)%}",
+                    r"{%[-+]? *end" + name + r"(?: .*?|)%}",
                     mode=DjTXT,
                     return_mode=self,
                 )
@@ -250,7 +250,7 @@ def create_token(self, raw_token, src, line):
         return token, mode
 
     def _has_closing_token(self, name, raw_token, src):
-        if not re.search(f"{{%[-\\+]? *end{name}(?: .*?|)%}}", src):
+        if not re.search(f"{{%[-+]? *end{name}(?: .*?|)%}}", src):
             return False
         if regex := self.AMBIGUOUS_BLOCK_TAGS.get(name):
             if regex[0]:
@@ -329,7 +329,7 @@ class DjCSS(DjTXT):
 
     RAW_TOKENS = DjTXT.RAW_TOKENS + [
         r"</style>",
-        r"[\{\(\)\}]",
+        r"[{()}]",
         r"/\*",
         r'".+"',
         r"'.+'",
@@ -379,12 +379,12 @@ class DjJS(DjTXT):
         r"\\.",
         r"//.*",
         r"/\*",
-        r"[\$\w-]+:",
+        r"[$\w-]+:",
         r'"(?:\\.|[^\\"])*"',  # "string"
         r"'(?:\\.|[^\\'])*'",  # 'string'
         r"`(?:\\.|[^\\`])*`",  # `string`
         r"`",
-        r"[\{\[\(\)\]\}]",
+        r"[{[()\]}]",
         r"var ",
         r"let ",
         r"const ",
@@ -399,8 +399,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.previous_offsets = []
         self.haskell = False
-        self.haskell_re = re.compile(r"^ *, ([\$\w-]+ *=|[\$\w-]+;?)")
-        self.variable_re = re.compile(r"^ *([\$\w-]+ *=|[\$\w-]+;?)")
+        self.haskell_re = re.compile(r"^ *, ([$\w-]+ *=|[$\w-]+;?)")
+        self.variable_re = re.compile(r"^ *([$\w-]+ *=|[$\w-]+;?)")
         self.previous_line_ended_with_comma = False
 
     def create_token(self, raw_token, src, line):

From 553e2543a54ecdd88462b0752f66151fc0dd731a Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 18:17:19 +0100
Subject: [PATCH 41/45] Use `!r` conversion when generating repr() strings in
 debug mode

---
 djhtml/lines.py  | 4 ++--
 djhtml/tokens.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/djhtml/lines.py b/djhtml/lines.py
index 010f0d9..03f8153 100644
--- a/djhtml/lines.py
+++ b/djhtml/lines.py
@@ -66,5 +66,5 @@ def __repr__(self):
         kwargs = ""
         for attr in ["level", "offset", "ignore"]:
             if value := getattr(self, attr):
-                kwargs += f", {attr}={value}"
-        return f"Line({repr(self.tokens)}{kwargs})"
+                kwargs += f", {attr}={value!r}"
+        return f"Line({self.tokens!r}{kwargs})"
diff --git a/djhtml/tokens.py b/djhtml/tokens.py
index 9304250..831fe7a 100644
--- a/djhtml/tokens.py
+++ b/djhtml/tokens.py
@@ -32,8 +32,8 @@ def __repr__(self):
             kwargs = f", mode={self.mode.__name__}"
             for attr in ["level", "relative", "absolute", "ignore"]:
                 if value := getattr(self, attr):
-                    kwargs += f", {attr}={value}"
-            return f"{self.__class__.__name__}('{self.text}'{kwargs})"
+                    kwargs += f", {attr}={value!r}"
+            return f"{self.__class__.__name__}({self.text!r}{kwargs})"
 
     class Text(_Base):
         pass

From 05a2de23dc3f62dac04cde9b5060beeed3daba2d Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 21:40:41 +0100
Subject: [PATCH 42/45] Preserve original indentation of {# comment #} tags
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On the one hand, we'd like a source like this:

    <div>
    <p>
    {# ignore me #}
    </p>
    </div>

to be indented to

    <div>
        <p>
            {# ignore me #}
        </p>
    </div>

and not to

    <div>
        <p>
    {# ignore me #}
        </p>
    </div>

But on the other hand, we'd like a source like this:

    <div>
    <p>
    {# ╔═══════════════════════════════╗
       ║ Carefully laid out table ║
       ╚═══════════════════════════════╝ #}
    </p>
    </div>

_not_ to get reindented to this:

    <div>
        <p>
            {# ╔═══════════════════════════════╗
            ║ Carefully laid out table ║
            ╚═══════════════════════════════╝ #}
        </p>
    </div>

In version <3, DjHTML compromised by indenting the starting token but
not the ending token of comments. However, that didn't make it better:

    <div>
        <p>
            {# ╔═══════════════════════════════╗
       ║ Carefully laid out table ║
       ╚═══════════════════════════════╝ #}
        </p>
    </div>

So, the only consistent thing to do is to preserve the existing
indentation of both opening and closing comment tags in all modes.
Note that this doesn't affect lines that don't start with a comment,
so the following:

    <div>
    <p>text</p>{# ignore me #}
    </div>

will still get reindented as:

    <div>
        <p>text</p>{# ignore me #}
    </div>

The new policy was already implemented for all comment tags except
{# comment #}. This commit concludes the transition to the new
behavior.
---
 djhtml/modes.py       | 10 +++-------
 tests/suite/html.html |  1 +
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index d837fc0..602d129 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -188,7 +188,6 @@ class DjTXT(BaseMode):
     RAW_TOKENS = [
         r"\n",
         r"{%[-+]?.*?[-+]?%}",
-        r"{#.*?#}",
         r"{#",
         r"{{.*?}}",
     ]
@@ -209,8 +208,6 @@ class DjTXT(BaseMode):
         "video": (" as ", None),
         "placeholder": (" or ", None),
     }
-    FMT_ON = r"{# fmt:on #}"
-    FMT_OFF = r"{# fmt:off #}"
     OPENING_TAG = r"{%[-+]? *(\w+).*?[-+]?%}"
 
     def create_token(self, raw_token, src, line):
@@ -236,14 +233,13 @@ def create_token(self, raw_token, src, line):
                 token = Token.Close(raw_token, mode=DjTXT, **self.offsets)
             else:
                 token = Token.Text(raw_token, mode=DjTXT, **self.offsets)
-        elif re.match(self.FMT_OFF, raw_token):
-            token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
-                self.FMT_ON, mode=DjTXT, return_mode=self
-            )
         elif raw_token == "{#":
             token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
+                r"\{# fmt:on #\}", mode=DjTXT, return_mode=self
+            ) if src.startswith(" fmt:off #}") else Comment(
                 r"#\}", mode=DjTXT, return_mode=self
             )
+
         else:
             token = Token.Text(raw_token, mode=self.__class__, **self.offsets)
 
diff --git a/tests/suite/html.html b/tests/suite/html.html
index bc143dc..9c355e0 100644
--- a/tests/suite/html.html
+++ b/tests/suite/html.html
@@ -109,6 +109,7 @@ <h1>Title</h1>
     </pre>
        {#
                  #}
+           {##}
 
 
 <!-- Real life example kindly provided by @jacklinke -->

From a7f9e0766418f25fdf3bd19215d731c12b88a6d2 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 23:23:18 +0100
Subject: [PATCH 43/45] Simplify regular expressions

---
 djhtml/modes.py | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index 602d129..f15417c 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -216,14 +216,8 @@ def create_token(self, raw_token, src, line):
         if tag := re.match(self.OPENING_TAG, raw_token):
             name = tag.group(1)
             if name in self.COMMENT_TAGS:
-                token, mode = Token.Open(
-                    raw_token,
-                    mode=DjTXT,
-                    ignore=True,
-                ), Comment(
-                    r"{%[-+]? *end" + name + r"(?: .*?|)%}",
-                    mode=DjTXT,
-                    return_mode=self,
+                token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
+                    "{% *end" + name + " *%}", mode=DjTXT, return_mode=self
                 )
             elif self._has_closing_token(name, raw_token, src):
                 token = Token.Open(raw_token, mode=DjTXT, **self.offsets)
@@ -235,9 +229,9 @@ def create_token(self, raw_token, src, line):
                 token = Token.Text(raw_token, mode=DjTXT, **self.offsets)
         elif raw_token == "{#":
             token, mode = Token.Open(raw_token, mode=DjTXT, ignore=True), Comment(
-                r"\{# fmt:on #\}", mode=DjTXT, return_mode=self
+                "{# fmt:on #}", mode=DjTXT, return_mode=self
             ) if src.startswith(" fmt:off #}") else Comment(
-                r"#\}", mode=DjTXT, return_mode=self
+                "#}", mode=DjTXT, return_mode=self
             )
 
         else:
@@ -379,7 +373,6 @@ class DjJS(DjTXT):
         r'"(?:\\.|[^\\"])*"',  # "string"
         r"'(?:\\.|[^\\'])*'",  # 'string'
         r"`(?:\\.|[^\\`])*`",  # `string`
-        r"`",
         r"[{[()\]}]",
         r"var ",
         r"let ",
@@ -425,10 +418,6 @@ def create_token(self, raw_token, src, line):
             if raw_token == ")":
                 persist_relative_offset = True
             token = Token.Close(raw_token, mode=DjJS)
-        elif raw_token == "`":
-            token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
-                "`", mode=DjJS, return_mode=self
-            )
         elif raw_token == "/*":
             token, mode = Token.Open(raw_token, mode=DjJS, ignore=True), Comment(
                 r"\*/", mode=DjJS, return_mode=self

From 01d6fdcab1cfafa0fe007f0f626029a038b05c9c Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 23:24:43 +0100
Subject: [PATCH 44/45] JavaScript regular expression literals

There is still one broken case that is _very_ hard to fix:

    var re = x / y // comment

Here the string "/ y /" is recognized as a literal while it is not. This
means the comment is not tokenized as a comment, and if it contains any
opening tokens the next line will be incorrectly indented. A working example
is provided in the unittests.
---
 djhtml/modes.py     |  2 +-
 tests/suite/js.html | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/djhtml/modes.py b/djhtml/modes.py
index f15417c..e2c8630 100644
--- a/djhtml/modes.py
+++ b/djhtml/modes.py
@@ -366,13 +366,13 @@ class DjJS(DjTXT):
     """
 
     RAW_TOKENS = DjTXT.RAW_TOKENS + [
-        r"\\.",
         r"//.*",
         r"/\*",
         r"[$\w-]+:",
         r'"(?:\\.|[^\\"])*"',  # "string"
         r"'(?:\\.|[^\\'])*'",  # 'string'
         r"`(?:\\.|[^\\`])*`",  # `string`
+        r"/(?:\\.|[^\\/\n])*/",  # /string/
         r"[{[()\]}]",
         r"var ",
         r"let ",
diff --git a/tests/suite/js.html b/tests/suite/js.html
index 0f29cb3..12d2f4b 100644
--- a/tests/suite/js.html
+++ b/tests/suite/js.html
@@ -436,3 +436,20 @@
             baz
     }
 </script>
+
+
+<!-- Expression literals -->
+<script>
+    re = /ab+c/
+    space_re = / foo /
+    single_re = /foo\/ + bar/
+    double_re = /foo/ + /bar/
+    look_like_comment = /\(//
+
+    // The following is known to be broken
+    const x = 42
+    const y = 7
+    const six = x / (function() { // comment
+    return y
+    })();
+</script>

From 8a8968611903806a6c49ad37ff8638979b064655 Mon Sep 17 00:00:00 2001
From: Jaap Joris Vens <jj@rtts.eu>
Date: Tue, 21 Feb 2023 23:57:25 +0100
Subject: [PATCH 45/45] Compare actual with expected tokenization in unittests

This guarantees that DjHTML is exactly behaving as expected, even if a
tokenization mishap doesn't result in changed indentation. An added bonus is
that by tracking these files in Git, we will notice changes in tokenization
immediately.

The initial token files were generated by the supplied `generate_tokens.py`
script.
---
 tests/generate_tokens.py     |  17 ++
 tests/suite/css.tokens       | 190 +++++++++++++++
 tests/suite/django.tokens    | 166 +++++++++++++
 tests/suite/html.tokens      | 355 ++++++++++++++++++++++++++++
 tests/suite/jinja.tokens     |  55 +++++
 tests/suite/js.tokens        | 447 +++++++++++++++++++++++++++++++++++
 tests/suite/paradoxes.tokens |  51 ++++
 tests/test_suite.py          |  22 +-
 8 files changed, 1296 insertions(+), 7 deletions(-)
 create mode 100755 tests/generate_tokens.py
 create mode 100644 tests/suite/css.tokens
 create mode 100644 tests/suite/django.tokens
 create mode 100644 tests/suite/html.tokens
 create mode 100644 tests/suite/jinja.tokens
 create mode 100644 tests/suite/js.tokens
 create mode 100644 tests/suite/paradoxes.tokens

diff --git a/tests/generate_tokens.py b/tests/generate_tokens.py
new file mode 100755
index 0000000..5ffb11f
--- /dev/null
+++ b/tests/generate_tokens.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+"""
+Generate *.tokens files from *.html files in the suite directory.
+
+"""
+
+from pathlib import Path
+
+from djhtml.modes import DjHTML
+
+DIR = Path(__file__).parent / "suite"
+
+for filename in DIR.iterdir():
+    if filename.suffix == ".html":
+        with open(DIR / filename) as html:
+            with open(DIR / (filename.stem + ".tokens"), "w") as f:
+                f.write(DjHTML(html.read()).debug())
diff --git a/tests/suite/css.tokens b/tests/suite/css.tokens
new file mode 100644
index 0000000..12da6cb
--- /dev/null
+++ b/tests/suite/css.tokens
@@ -0,0 +1,190 @@
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Basic CSS ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    a ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('text-decoration: ', mode=DjCSS), Text('none', mode=DjCSS, absolute=17), Text(';', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Nested SCSS ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    a ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('text-decoration: ', mode=DjCSS), Text('none', mode=DjCSS, absolute=17), Text(';', mode=DjCSS)], level=2)
+Line([Text('        &:hover ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('text-decoration: ', mode=DjCSS), Text('underline', mode=DjCSS, absolute=17), Text(';', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multiline comment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjCSS), Open('/*', mode=DjCSS, level=1, ignore=True)], level=1, ignore=True)
+Line([Text('     * Hi mom!', mode=Comment, ignore=True)], level=2, ignore=True)
+Line([Text('     *', mode=Comment, ignore=True)], level=2, ignore=True)
+Line([Text('     ', mode=Comment, ignore=True), Close('*/', mode=DjCSS, ignore=True)], level=1, ignore=True)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' CSS Function ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    #gradient ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('background: ', mode=DjCSS), Text('linear-gradient', mode=DjCSS, absolute=12), Open('(', mode=DjCSS, level=2, absolute=12)], level=2)
+Line([Text('                        to top left,', mode=DjCSS, absolute=12)], level=3, offset=12)
+Line([Text('                        rgba', mode=DjCSS, absolute=12), Open('(', mode=DjCSS, level=3, absolute=12), Text('204,0,0,1', mode=DjCSS, absolute=12), Close(')', mode=DjCSS, absolute=12), Text(',', mode=DjCSS, absolute=12)], level=3, offset=12)
+Line([Text('                        rgba', mode=DjCSS, absolute=12), Open('(', mode=DjCSS, level=3, absolute=12), Text('204,0,0,0', mode=DjCSS, absolute=12), Close(')', mode=DjCSS, absolute=12)], level=3, offset=12)
+Line([Text('                    ', mode=DjCSS, absolute=12), Close(')', mode=DjCSS, absolute=12), Text(';', mode=DjCSS)], level=2, offset=12)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' @font-face typical usage ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    @font-face ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('font-family: ', mode=DjCSS), Text("'Open Sans'", mode=DjCSS, absolute=13), Text(';', mode=DjCSS)], level=2)
+Line([Text('        ', mode=DjCSS), Text('font-style: ', mode=DjCSS), Text('normal', mode=DjCSS, absolute=12), Text(';', mode=DjCSS)], level=2)
+Line([Text('        ', mode=DjCSS), Text('font-weight: ', mode=DjCSS), Text('normal', mode=DjCSS, absolute=13), Text(';', mode=DjCSS)], level=2)
+Line([Text('        ', mode=DjCSS), Text('src: ', mode=DjCSS), Text('url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Open_Sans.woff2') format('woff2'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(',', mode=DjCSS, absolute=5)], level=2)
+Line([Text('             url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Open_Sans.woff') format('woff'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(',', mode=DjCSS, absolute=5)], level=2, offset=5)
+Line([Text('             url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Open_Sans.eot?#iefix') format('embedded-opentype'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(',', mode=DjCSS, absolute=5)], level=2, offset=5)
+Line([Text('             url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Open_Sans.svg#OpenSans') format('svg'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(',', mode=DjCSS, absolute=5)], level=2, offset=5)
+Line([Text('             url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Open_Sans.ttf') format('truetype'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(';', mode=DjCSS)], level=2, offset=5)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The example from the README ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    @font-face ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('font-family: ', mode=DjCSS), Text('Helvetica', mode=DjCSS, absolute=13), Text(';', mode=DjCSS)], level=2)
+Line([Text('        ', mode=DjCSS), Text('src: ', mode=DjCSS), Open('{% for format, filename in licensed_fonts %}', mode=DjTXT, level=2, absolute=5)], level=2)
+Line([Text('                 url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=3, absolute=5), Text("'{% static filename %}') format('{{ format }}'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(',', mode=DjCSS, absolute=5)], level=3, offset=5)
+Line([Text('             ', mode=DjCSS, absolute=5), Close('{% endfor %}', mode=DjTXT, absolute=5)], level=2, offset=5)
+Line([Text('             url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Arial.woff2') format('woff2'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(',', mode=DjCSS, absolute=5)], level=2, offset=5)
+Line([Text('             url', mode=DjCSS, absolute=5), Open('(', mode=DjCSS, level=2, absolute=5), Text("'Arial.woff') format('woff'", mode=DjCSS, absolute=5), Close(')', mode=DjCSS, absolute=5), Text(';', mode=DjCSS)], level=2, offset=5)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' @media ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    @media', mode=DjCSS), Open('(', mode=DjCSS, level=1), Text('max-width: ', mode=DjCSS), Text('800px', mode=DjCSS, absolute=18), Close(')', mode=DjCSS, absolute=18), Text(' ', mode=DjCSS, absolute=18), Open('{', mode=DjCSS, level=1, absolute=18)], level=1)
+Line([Text('        body ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('margin-top: ', mode=DjCSS), Text('1em', mode=DjCSS, absolute=12), Text(';', mode=DjCSS)], level=3)
+Line([Text('            ', mode=DjCSS), Text('margin-bottom: ', mode=DjCSS), Text('1em', mode=DjCSS, absolute=15), Text(';', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Missing semicolons ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    p ', mode=DjCSS), Open('{', mode=DjCSS, level=1), Text(' ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left ', mode=DjCSS, absolute=11), Close('}', mode=DjCSS)], level=1)
+Line([Text('    h1 ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('font-weight: ', mode=DjCSS), Text('bold', mode=DjCSS, absolute=13), Text(';', mode=DjCSS)], level=2)
+Line([Text('        ', mode=DjCSS), Text('font-size: ', mode=DjCSS), Text('huge', mode=DjCSS, absolute=11)], level=2)
+Line([Text('    ', mode=DjCSS, absolute=11), Close('}', mode=DjCSS)], level=1)
+Line([Text('    footnote ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('dontread: ', mode=DjCSS), Text('true !important', mode=DjCSS, absolute=10)], level=2)
+Line([Text('    ', mode=DjCSS, absolute=10), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Nested SCSS with missing semicolons and different indentation styles ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    article ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        p ', mode=DjCSS), Open('{', mode=DjCSS, level=2), Text(' ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('#999', mode=DjCSS, absolute=11), Text(';', mode=DjCSS), Text(' ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left ', mode=DjCSS, absolute=24), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Text('    article ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        p ', mode=DjCSS), Open('{', mode=DjCSS, level=2), Text(' ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('#999', mode=DjCSS, absolute=11), Text(';', mode=DjCSS)], level=2)
+Line([Text('            ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left ', mode=DjCSS, absolute=7), Close('}', mode=DjCSS)], level=3)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Text('    article ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        p ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('#999', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('            ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left ', mode=DjCSS, absolute=7), Close('}', mode=DjCSS)], level=3)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Text('    article ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        p ', mode=DjCSS), Open('{', mode=DjCSS, level=2), Text(' ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('#999', mode=DjCSS, absolute=11), Text(';', mode=DjCSS)], level=2)
+Line([Text('            ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left', mode=DjCSS, absolute=7)], level=3)
+Line([Text('        ', mode=DjCSS, absolute=7), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Text('    article ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        p ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('#999', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('            ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left', mode=DjCSS, absolute=7)], level=3)
+Line([Text('        ', mode=DjCSS, absolute=7), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Text('    article ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        p', mode=DjCSS)], level=2)
+Line([Text('        ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('#999', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('            ', mode=DjCSS), Text('align: ', mode=DjCSS), Text('left', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Selector weirdness ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    blockquote:before ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('content: ', mode=DjCSS), Text('"text-align: right"', mode=DjCSS, absolute=9), Text(';', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Text('    @media', mode=DjCSS), Open('(', mode=DjCSS, level=1), Text('text-align: ', mode=DjCSS), Text('right', mode=DjCSS, absolute=19), Close(')', mode=DjCSS, absolute=19), Text(' ', mode=DjCSS, absolute=19), Open('{', mode=DjCSS, level=1, absolute=19)], level=1)
+Line([Text('        a:hover, a:active ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            text-decoration:none', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('}', mode=DjCSS)], level=2)
+Line([Text('        p::first-line ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('red', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Nested SCSS with multi-line statements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    a:hover, a:active ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('foo: ', mode=DjCSS), Text('bar,', mode=DjCSS, absolute=5)], level=2)
+Line([Text('             baz', mode=DjCSS, absolute=5), Text(';', mode=DjCSS)], level=2, offset=5)
+Line([])
+Line([Text('        a:hover, a:active ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('foo: ', mode=DjCSS), Text('bar', mode=DjCSS, absolute=5)], level=3)
+Line([Text('                 baz', mode=DjCSS, absolute=5)], level=3, offset=5)
+Line([Text('        ', mode=DjCSS, absolute=5), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The difference between multi-line statements and nested selectors ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    body ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('font-family: ', mode=DjCSS), Text('Arial,', mode=DjCSS, absolute=13)], level=2)
+Line([Text('                     Helvetica', mode=DjCSS, absolute=13), Text(';', mode=DjCSS)], level=2, offset=13)
+Line([Text('        a:link,', mode=DjCSS)], level=2)
+Line([Text('        a:hover,', mode=DjCSS)], level=2)
+Line([Text('        a:active ', mode=DjCSS), Open('{', mode=DjCSS, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('red', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('}', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([])
+Line([Text('    ', mode=DjCSS), Open('/*', mode=DjCSS, level=1, ignore=True), Text(' Since DjCSS expects a space between the property and the value,', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('       the following is currently broken ', mode=Comment, ignore=True), Close('*/', mode=DjCSS, ignore=True)], level=2, ignore=True)
+Line([Text('    body ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        font-family:Arial,', mode=DjCSS)], level=2)
+Line([Text('        Helvetica', mode=DjCSS), Text(';', mode=DjCSS)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line statements with prior statements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    body ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('green', mode=DjCSS, absolute=7), Text(';', mode=DjCSS), Text(' ', mode=DjCSS), Text('font-family: ', mode=DjCSS), Text('Arial,', mode=DjCSS, absolute=27)], level=2)
+Line([Text('                                   Helvetica', mode=DjCSS, absolute=27), Text(';', mode=DjCSS)], level=2, offset=27)
+Line([])
+Line([Text('        ', mode=DjCSS), Open('/*', mode=DjCSS, level=2, ignore=True), Text(' Spaces within lines are always preserved ', mode=Comment, ignore=True), Close('*/', mode=DjCSS, ignore=True)], level=2, ignore=True)
+Line([Text('        ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('red', mode=DjCSS, absolute=7), Text(';', mode=DjCSS), Text('   ', mode=DjCSS), Text('font-family: ', mode=DjCSS), Text('Arial,', mode=DjCSS, absolute=27)], level=2)
+Line([Text('                                   Helvetica', mode=DjCSS, absolute=27), Text(';', mode=DjCSS)], level=2, offset=27)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
\ No newline at end of file
diff --git a/tests/suite/django.tokens b/tests/suite/django.tokens
new file mode 100644
index 0000000..6a7cf0a
--- /dev/null
+++ b/tests/suite/django.tokens
@@ -0,0 +1,166 @@
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The example from the README ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12)])
+Line([Text('    ', mode=InsideHTMLTag, absolute=12), Open('{#', mode=DjTXT, ignore=True), Text(' fmt:off #}', mode=Comment, ignore=True)], ignore=True)
+Line([Text('      ,-._|\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     /     .\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     \\_,--._/', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=Comment, ignore=True), Close('{# fmt:on #}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('           ', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Text('/>', mode=DjHTML)], offset=11)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Basic everyday Django template ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('{% load static %}', mode=DjTXT)])
+Line([Open('{% block content %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Open('{% if request.user.is_authenticated %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('h1', mode=InsideHTMLTag, absolute=4), Open('>', mode=DjHTML, level=2), Text('Welcome, ', mode=DjHTML), Text('{{ request.user }}', mode=DjHTML), Close('</h1>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), CloseAndOpen('{% else %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text("{% url 'login' %}", mode=DjTXT, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=2), Text('Click here to login', mode=DjHTML), Close('</a>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Close('{% endblock %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Django tags within <style> ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    a ', mode=DjCSS), Open('{', mode=DjCSS, level=1)], level=1)
+Line([Text('        ', mode=DjCSS), Open('{% block color %}', mode=DjTXT, level=2)], level=2)
+Line([Text('            ', mode=DjCSS), Text('color: ', mode=DjCSS), Text('red', mode=DjCSS, absolute=7), Text(';', mode=DjCSS)], level=3)
+Line([Text('        ', mode=DjCSS), Close('{% endblock %}', mode=DjTXT)], level=2)
+Line([Text('    ', mode=DjCSS), Close('}', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Django tags within <script> ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    $', mode=DjJS), Open('(', mode=DjJS, level=1), Text('function', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        ', mode=DjJS), Open('{% if verbose %}', mode=DjTXT, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text('"Hi mom!"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('{% endif %}', mode=DjTXT)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multiline comment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% comment %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('      ,-._|\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     /     .\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     \\_,--._/', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('{% endcomment %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Verbatim block ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% verbatim %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('      ,-._|\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     /     .\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     \\_,--._/', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('{% endverbatim %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multiline comment with optional comment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% comment "Yo dawg" %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('            I heard you like comments', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('{% endcomment %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Various spacings ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{%comment    %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('             %}', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('{%    endcomment%}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Open('{%if%}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Open('{%block content    %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        Yuck!', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{%    endblock%}', mode=DjTXT)], level=1)
+Line([Close('{%    endif    %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Inconsistent nesting ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% block %}', mode=DjTXT)])
+Line([Text('    1', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Open('{% with %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        2', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endblock %}', mode=DjTXT)], level=1)
+Line([Text('    1', mode=DjHTML)], level=1)
+Line([Close('{% endwith %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Tags that both dedent and indent ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% blocktrans count count=queryset.count %}', mode=DjTXT)])
+Line([Text('    There is 1 item.', mode=DjHTML)], level=1)
+Line([CloseAndOpen('{% plural %}', mode=DjTXT)])
+Line([Text('    There are ', mode=DjHTML), Text('{{ count }}', mode=DjHTML), Text(' items.', mode=DjHTML)], level=1)
+Line([Close('{% endblocktrans %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' None of this should be indented ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% if %}', mode=DjTXT)])
+Line([CloseAndOpen('{% elif %}', mode=DjTXT)])
+Line([CloseAndOpen('{% else %}', mode=DjTXT)])
+Line([Close('{% endif %}', mode=DjTXT)])
+Line([Open('{% for %}', mode=DjTXT)])
+Line([CloseAndOpen('{% empty %}', mode=DjTXT)])
+Line([Close('{% endfor %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Template tag directly following tag name ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3), Open('{% if %}', mode=DjTXT, absolute=3), Text(' ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=18), Text('"', mode=InsideHTMLTag, absolute=17), Close('{% endif %}', mode=DjTXT, absolute=3), Open('>', mode=DjHTML)])
+Line([Text('    Click here!', mode=DjHTML)], level=1)
+Line([Close('</a>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Ambiguous video tag from django-embed-video ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% video form.instance.video as my_video %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text("{% video my_video '360 x 200' %}", mode=DjTXT)], level=1)
+Line([Close('{% endvideo %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Match the tag, the whole tag, and nothing but the tag ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% component_block "instruction" border_top=True header="Login" %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Open('{% slot "body" %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Open('{% with "login --token="|add:token as content %}', mode=DjTXT, level=2)], level=2)
+Line([Text('            ', mode=DjHTML), Text('{% component "copy_field" field_selector="token-field" content=content %}', mode=DjTXT)], level=3)
+Line([Text('        ', mode=DjHTML), Close('{% endwith %}', mode=DjTXT)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endslot %}', mode=DjTXT)], level=1)
+Line([Close('{% endcomponent_block %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(" Don't indent template variables ", mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text("{{ '<b>' if condition }}", mode=DjHTML)])
+Line([Text('some text', mode=DjHTML)])
+Line([Text("{{ '</b>' if condition }}", mode=DjHTML)])
+Line([Text('more text', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Unless they span multiple lines ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text("{{ '", mode=DjHTML), Text('<', mode=DjHTML), Text('b', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML), Text("'", mode=DjHTML)])
+Line([Text('    if condition }}', mode=DjHTML)], level=1)
+Line([Text('    some text', mode=DjHTML)], level=1)
+Line([Text("    {{ '", mode=DjHTML), Close('</b>', mode=DjHTML)], level=1)
+Line([Text('if condition }}', mode=DjHTML)])
+Line([Text('more text', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The example from issue #78 ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% block main %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('section', mode=InsideHTMLTag, absolute=9), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=3)], level=3)
+Line([Text('                ', mode=DjHTML), Text('{% placeholder "placeholder1" %}', mode=DjTXT)], level=4)
+Line([Text('            ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=3)
+Line([Text('            ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=3)], level=3)
+Line([Text('                ', mode=DjHTML), Open('{% placeholder "placeholder2" or %}', mode=DjTXT, level=4)], level=4)
+Line([Text('                    ', mode=DjHTML), Text('<', mode=DjHTML), Text('img ', mode=InsideHTMLTag, absolute=5), Text('src=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9), Text(' ', mode=InsideHTMLTag, absolute=5), Text('alt=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=17), Text('"', mode=InsideHTMLTag, absolute=16), Text(' ', mode=InsideHTMLTag, absolute=5), Text('/>', mode=DjHTML)], level=5)
+Line([Text('                ', mode=DjHTML), Close('{% endplaceholder %}', mode=DjTXT)], level=4)
+Line([Text('            ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</section>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('section', mode=InsideHTMLTag, absolute=9), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            ', mode=DjHTML), Text('{% placeholder "placeholder3" %}', mode=DjTXT)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</section>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('section', mode=InsideHTMLTag, absolute=9), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            ', mode=DjHTML), Text('{% placeholder "placeholder4" %}', mode=DjTXT)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</section>', mode=DjHTML)], level=1)
+Line([Close('{% endblock %}', mode=DjTXT)])
+Line([])
\ No newline at end of file
diff --git a/tests/suite/html.tokens b/tests/suite/html.tokens
new file mode 100644
index 0000000..e66c65f
--- /dev/null
+++ b/tests/suite/html.tokens
@@ -0,0 +1,355 @@
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Basic HTML sanity check ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('!doctype html>', mode=DjHTML)])
+Line([Text('<', mode=DjHTML), Text('html', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('head', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('title', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML, level=2), Text('Plain', mode=DjHTML), Close('</title>', mode=DjHTML)], level=2)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('script ', mode=InsideHTMLTag, absolute=8), Text('src=', mode=InsideHTMLTag, absolute=8), Text('"', mode=InsideHTMLTag, absolute=13), Text('test.js', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12), Open('>', mode=DjHTML, level=2), Close('</script>', mode=DjHTML)], level=2)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('link ', mode=InsideHTMLTag, absolute=6), Text('rel=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=11), Text('stylesheet', mode=InsideHTMLTag, absolute=11), Text('"', mode=InsideHTMLTag, absolute=10), Text(' ', mode=InsideHTMLTag, absolute=6), Text('href=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=29), Text('test.css', mode=InsideHTMLTag, absolute=29), Text('"', mode=InsideHTMLTag, absolute=28), Text('>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</head>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('body', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('h1', mode=InsideHTMLTag, absolute=4), Open('>', mode=DjHTML, level=2), Text('Title', mode=DjHTML), Close('</h1>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</body>', mode=DjHTML)], level=1)
+Line([Close('</html>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multiple closing/opening tags on same line ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=11), Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Some crazy edgecases that even the mighty web-mode fails to indent correctly ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            text', mode=DjHTML)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML), Close('</div>', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([])
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, relative=-1, absolute=10), Open('>', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, relative=-1, absolute=15), Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([Close('</div>', mode=DjHTML)])
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('div0', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML), Text('<', mode=DjHTML), Text('div1', mode=InsideHTMLTag, relative=-1, absolute=12), Open('>', mode=DjHTML)])
+Line([Text('    text2', mode=DjHTML), Close('</div1>', mode=DjHTML), Text('<', mode=DjHTML), Text('div1', mode=InsideHTMLTag, absolute=18), Open('>', mode=DjHTML, level=1), Text('<', mode=DjHTML), Text('div2', mode=InsideHTMLTag, absolute=24), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        text3', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</div2>', mode=DjHTML), Text('<', mode=DjHTML), Text('div2', mode=InsideHTMLTag, absolute=13), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        text3', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</div2>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Close('</div1>', mode=DjHTML)], level=1)
+Line([Close('</div0>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('div0', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div1', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div2', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            text3', mode=DjHTML)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div2>', mode=DjHTML), Close('</div1>', mode=DjHTML), Close('</div0>', mode=DjHTML), Text('<', mode=DjHTML), Text('div0', mode=InsideHTMLTag, absolute=27), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            text1', mode=DjHTML)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div0>', mode=DjHTML)], level=2)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Ignore self-closing tags ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Text('/>', mode=DjHTML)], level=1)
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Ignore void tags ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('img', mode=InsideHTMLTag, absolute=5), Text('>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('input', mode=InsideHTMLTag, absolute=7), Text('>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('input ', mode=InsideHTMLTag, absolute=7), Text('/>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('input', mode=InsideHTMLTag, absolute=7), Text('>', mode=DjHTML), Text('</input>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('!doctype>', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(" Don't parse style/script as html ", mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    <ul>', mode=DjJS)], level=1)
+Line([Text('    <li>', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('style', mode=InsideHTMLTag, absolute=7), Open('>', mode=DjHTML)])
+Line([Text('    <style>', mode=DjCSS)], level=1)
+Line([Close('</style>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Except for JS templates... ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script ', mode=InsideHTMLTag, absolute=8), Text('id=', mode=InsideHTMLTag, absolute=8), Text('"', mode=InsideHTMLTag, absolute=12), Text('id', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Text(' ', mode=InsideHTMLTag, absolute=8), Text('type=', mode=InsideHTMLTag, absolute=8), Text('"', mode=InsideHTMLTag, absolute=22), Text('text/template', mode=InsideHTMLTag, absolute=22), Text('"', mode=InsideHTMLTag, absolute=21), Text(' ', mode=InsideHTMLTag, absolute=8), Text('class=', mode=InsideHTMLTag, absolute=8), Text('"', mode=InsideHTMLTag, absolute=44), Text('class', mode=InsideHTMLTag, absolute=44), Text('"', mode=InsideHTMLTag, absolute=43), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('ul', mode=InsideHTMLTag, absolute=4), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('li', mode=InsideHTMLTag, absolute=4), Text('/>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</ul>', mode=DjHTML)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Ignore multiline comments ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('<!--', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text(' 1', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('  2', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('   3', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Ignore <pre> elements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('<pre>', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('   <dl>', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('  <dt>', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text(' <dd>', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('</pre>', mode=DjHTML, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Ignore both begin and end tags ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('    ', mode=DjHTML), Open('<!--', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text(' ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('            ', mode=DjHTML), Open('<pre>', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('    ', mode=Comment, ignore=True), Close('</pre>', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('       ', mode=DjHTML), Open('{#', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('                 ', mode=Comment, ignore=True), Close('#}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('           ', mode=DjHTML), Open('{#', mode=DjTXT, ignore=True), Close('#}', mode=DjTXT, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Real life example kindly provided by @jacklinke ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('row', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('input-group date', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Text(' ', mode=InsideHTMLTag, absolute=5), Text('data-provide=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=44), Text('datepicker', mode=InsideHTMLTag, absolute=44), Text('"', mode=InsideHTMLTag, absolute=43), Text(' ', mode=InsideHTMLTag, absolute=5), Text('data-date-format=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=74), Text('yyyy/mm/dd', mode=InsideHTMLTag, absolute=74), Text('"', mode=InsideHTMLTag, absolute=73), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('btn', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([Close('</div>', mode=DjHTML), Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=11), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('custom-control custom-switch', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('input ', mode=InsideHTMLTag, absolute=7), Text('type=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=13), Text('checkbox', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12), Text(' ', mode=InsideHTMLTag, absolute=7), Text('name=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=29), Text('filter-changes', mode=InsideHTMLTag, absolute=29), Text('"', mode=InsideHTMLTag, absolute=28), Text(' ', mode=InsideHTMLTag, absolute=7), Text('id=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=49), Text('filter-changes', mode=InsideHTMLTag, absolute=49), Text('"', mode=InsideHTMLTag, absolute=48), Text(' ', mode=InsideHTMLTag, absolute=7), Text('class=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=72), Text('custom-control-input fs-nano', mode=InsideHTMLTag, absolute=72), Text('"', mode=InsideHTMLTag, absolute=71), Text(' ', mode=InsideHTMLTag, absolute=7), Text('value=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=109), Text('0', mode=InsideHTMLTag, absolute=109), Text('"', mode=InsideHTMLTag, absolute=108), Text(' ', mode=InsideHTMLTag, absolute=7), Open('{% if request.session.manage_filter_changes %}', mode=DjTXT, level=2, absolute=7), Open('{% if request.session.manage_filter_changes == True %}', mode=DjTXT, level=2, absolute=7), Text('checked', mode=InsideHTMLTag, absolute=7), Close('{% endif %}', mode=DjTXT, absolute=7), Close('{% endif %}', mode=DjTXT, absolute=7), Text('>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('form ', mode=InsideHTMLTag, absolute=6), Text('class=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=13), Text('form-inline', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('custom-control custom-switch', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            ', mode=DjHTML), Text('<', mode=DjHTML), Text('input ', mode=InsideHTMLTag, absolute=7), Text('type=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=13), Text('checkbox', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12), Text(' ', mode=InsideHTMLTag, absolute=7), Text('name=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=29), Text('filter-status-new', mode=InsideHTMLTag, absolute=29), Text('"', mode=InsideHTMLTag, absolute=28), Text(' ', mode=InsideHTMLTag, absolute=7), Text('id=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=52), Text('filter-status-new', mode=InsideHTMLTag, absolute=52), Text('"', mode=InsideHTMLTag, absolute=51), Text(' ', mode=InsideHTMLTag, absolute=7), Text('class=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=78), Text('custom-control-input fs-nano', mode=InsideHTMLTag, absolute=78), Text('"', mode=InsideHTMLTag, absolute=77), Text(' ', mode=InsideHTMLTag, absolute=7), Text('value=', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=115), Text('0', mode=InsideHTMLTag, absolute=115), Text('"', mode=InsideHTMLTag, absolute=114), Text(' ', mode=InsideHTMLTag, absolute=7), Open('{% if request.session.manage_filter_status_new %}', mode=DjTXT, level=3, absolute=7), Open('{% if request.session.manage_filter_status_new == True %}', mode=DjTXT, level=3, absolute=7), Text('checked', mode=InsideHTMLTag, absolute=7), Close('{% endif %}', mode=DjTXT, absolute=7), Close('{% endif %}', mode=DjTXT, absolute=7), Text('>', mode=DjHTML)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</form>', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' SVG with multi-line attribute values ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('svg ', mode=InsideHTMLTag, absolute=5), Text('width=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('325', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11), Text(' ', mode=InsideHTMLTag, absolute=5), Text('height=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=25), Text('325', mode=InsideHTMLTag, absolute=25), Text('"', mode=InsideHTMLTag, absolute=24), Text(' ', mode=InsideHTMLTag, absolute=5), Text('xmlns=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=37), Text('http://www.w3.org/2000/svg', mode=InsideHTMLTag, absolute=37), Text('"', mode=InsideHTMLTag, absolute=36), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('path', mode=InsideHTMLTag, absolute=6)], level=1)
+Line([Text('          ', mode=InsideHTMLTag, absolute=6), Text('d=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=9), Text('M 80 80', mode=InsideHTMLTag, absolute=9)], level=1, offset=6)
+Line([Text('             A 45 45, 0, 0, 0, 125 125', mode=InsideHTMLTag, absolute=9)], level=1, offset=9)
+Line([Text('             L 125 80 Z', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)], level=1, offset=9)
+Line([Text('          ', mode=InsideHTMLTag, absolute=6), Text('fill=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=12), Text('green', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11)], level=1, offset=6)
+Line([Text('    ', mode=InsideHTMLTag, absolute=6), Text('/>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('path', mode=InsideHTMLTag, absolute=6)], level=1)
+Line([Text('          ', mode=InsideHTMLTag, absolute=6), Text('d=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=9)], level=1, offset=6)
+Line([Text('             M 80 80', mode=InsideHTMLTag, absolute=9)], level=1, offset=9)
+Line([Text('             A 45 45, 0, 0, 0, 125 125', mode=InsideHTMLTag, absolute=9)], level=1, offset=9)
+Line([Text('             L 125 80 Z', mode=InsideHTMLTag, absolute=9)], level=1, offset=9)
+Line([Text('            ', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)], level=1, offset=8)
+Line([Text('          ', mode=InsideHTMLTag, absolute=6), Text('fill=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=12), Text('green', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11)], level=1, offset=6)
+Line([Text('    ', mode=InsideHTMLTag, absolute=6), Text('/>', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('path ', mode=InsideHTMLTag, absolute=6), Text('d=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=9), Text('M 80 80', mode=InsideHTMLTag, absolute=9)], level=1)
+Line([Text('             A 45 45, 0, 0, 0, 125 125', mode=InsideHTMLTag, absolute=9)], level=1, offset=9)
+Line([Text('             L 125 80 Z', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)], level=1, offset=9)
+Line([Text('          ', mode=InsideHTMLTag, absolute=6), Text('fill=', mode=InsideHTMLTag, absolute=6), Text('"', mode=InsideHTMLTag, absolute=12), Text('green', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11)], level=1, offset=6)
+Line([Text('    ', mode=InsideHTMLTag, absolute=6), Text('/>', mode=DjHTML)], level=1)
+Line([Close('</svg>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Greater than character inside tag ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Open('{% if foo > bar %}', mode=DjTXT, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=30), Text('baz', mode=InsideHTMLTag, absolute=30), Text('"', mode=InsideHTMLTag, absolute=29), Close('{% endif %}', mode=DjTXT, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    quux', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Block tags inside HTML elements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5)])
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Open('{% block class %}', mode=DjTXT, absolute=5)], offset=5)
+Line([Text('         ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12), Text('foo', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11)], level=1, offset=5)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Close('{% endblock %}', mode=DjTXT, absolute=5)], offset=5)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5)])
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Open('{% block classes %}', mode=DjTXT, absolute=5)], offset=5)
+Line([Text('         ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=12)], level=1, offset=5)
+Line([Text('                foo', mode=InsideHTMLTag, absolute=12)], level=1, offset=12)
+Line([Text('                bar', mode=InsideHTMLTag, absolute=12)], level=1, offset=12)
+Line([Text('               ', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=11)], level=1, offset=11)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Close('{% endblock %}', mode=DjTXT, absolute=5)], offset=5)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9), Text('1', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Text(' ', mode=InsideHTMLTag, absolute=5), Text('class=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=19), Open('{% block classes %}', mode=DjTXT, absolute=19)])
+Line([Text('                       foo', mode=InsideHTMLTag, absolute=19)], level=1, offset=19)
+Line([Text('                       bar', mode=InsideHTMLTag, absolute=19)], level=1, offset=19)
+Line([Text('                       ', mode=InsideHTMLTag, absolute=19), Open('{% if %}', mode=DjTXT, level=1, absolute=19)], level=1, offset=19)
+Line([Text('                           baz', mode=InsideHTMLTag, absolute=19)], level=2, offset=19)
+Line([Text('                       ', mode=InsideHTMLTag, absolute=19), CloseAndOpen('{% elif %}', mode=DjTXT, level=1, absolute=19)], level=1, offset=19)
+Line([Text('                           quux', mode=InsideHTMLTag, absolute=19)], level=2, offset=19)
+Line([Text('                       ', mode=InsideHTMLTag, absolute=19), Close('{% endif %}', mode=DjTXT, absolute=19)], level=1, offset=19)
+Line([Text('                   ', mode=InsideHTMLTag, absolute=19), Close('{% endblock %}', mode=DjTXT, absolute=19)], offset=19)
+Line([Text('                  ', mode=InsideHTMLTag, absolute=19), Text('"', mode=InsideHTMLTag, absolute=18)], offset=18)
+Line([Text('     hidden', mode=InsideHTMLTag, absolute=5)], offset=5)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Non-block tags inside HTML elements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('{% url %}', mode=DjTXT, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=10), Text('button', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9)], offset=3)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    Click me!', mode=DjHTML)], level=1)
+Line([Close('</a>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Spurious < and > characters ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    >', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Arrow inside attribute ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('button ', mode=InsideHTMLTag, absolute=8), Text('action=', mode=InsideHTMLTag, absolute=8), Text('"', mode=InsideHTMLTag, absolute=16), Text('a-', mode=InsideHTMLTag, absolute=16), Text('>', mode=InsideHTMLTag, absolute=16), Text('b', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        hi', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</button>', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line elements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('id=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=7), Text('1', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=6), Text(' ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=10), Text('button', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9), Open('>', mode=DjHTML)], offset=3)
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</a>', mode=DjHTML)])
+Line([Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('id=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=7), Text('2', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=6), Text(' ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15)], offset=3)
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=10), Text('button', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9), Open('>', mode=DjHTML)], offset=3)
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</a>', mode=DjHTML)])
+Line([Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('id=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=7), Text('3', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=6), Text(' ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=10), Text('button', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9)], offset=3)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</a>', mode=DjHTML)])
+Line([Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('id=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=7), Text('4', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=6), Text(' ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15)], offset=3)
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=10), Text('button', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9)], offset=3)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</a>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The example from issue #50 ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('img ', mode=InsideHTMLTag, absolute=5), Text('src=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=10), Text('/some/long/path', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9)])
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9), Text('1', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Text('>', mode=DjHTML)], offset=5)
+Line([Text('<', mode=DjHTML), Text('blockquote ', mode=InsideHTMLTag, absolute=12), Text('class=', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=19), Text('dishy', mode=InsideHTMLTag, absolute=19), Text('"', mode=InsideHTMLTag, absolute=18)])
+Line([Text('            ', mode=InsideHTMLTag, absolute=12), Text('id=', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=16), Text('2', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15), Text(' ', mode=InsideHTMLTag, absolute=12), Text('/>', mode=DjHTML)], offset=12)
+Line([Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('/some/long/path', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('id=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=7), Text('3', mode=InsideHTMLTag, absolute=7), Text('"', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML), Close('</a>', mode=DjHTML)], offset=3)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line attribute values ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9)])
+Line([Text('         1', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)], offset=9)
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9), Text('2', mode=InsideHTMLTag, absolute=9)], level=1)
+Line([Text('            ', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=1)], level=1, offset=8)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9)], level=2)
+Line([Text('                 3', mode=InsideHTMLTag, absolute=9)], level=2, offset=9)
+Line([Text('                ', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=2)], level=2, offset=8)
+Line([Text('            ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9)], level=3)
+Line([Text('                     4', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)], level=3, offset=9)
+Line([Text('            ', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=3)], level=3)
+Line([Text('                ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9), Text('5', mode=InsideHTMLTag, absolute=9)], level=4)
+Line([Text('                        ', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)], level=4, offset=8)
+Line([Text('                ', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=4)], level=4)
+Line([Text('                    ', mode=DjHTML), Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('id=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=9)], level=5)
+Line([Text('                             6', mode=InsideHTMLTag, absolute=9)], level=5, offset=9)
+Line([Text('                            ', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)], level=5, offset=8)
+Line([Text('                    ', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML, level=5)], level=5)
+Line([Text('                    ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=5)
+Line([Text('                ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=4)
+Line([Text('            ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=3)
+Line([Text('        ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('</div>', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Weird but legal attribute names ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div ', mode=InsideHTMLTag, absolute=5), Text('!click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)])
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('@click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('#click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('$click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('%click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('^click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('&click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('*click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('<click=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=13)], offset=5)
+Line([Text('            ', mode=InsideHTMLTag, absolute=13), Text('"', mode=InsideHTMLTag, absolute=12)], offset=12)
+Line([Text('     ', mode=InsideHTMLTag, absolute=5), Text('data-data=', mode=InsideHTMLTag, absolute=5), Text('"', mode=InsideHTMLTag, absolute=16)], offset=5)
+Line([Text('               ', mode=InsideHTMLTag, absolute=16), Text('"', mode=InsideHTMLTag, absolute=15)], offset=15)
+Line([Open('>', mode=DjHTML)])
+Line([Text('    text', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Weird but legal element names ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('my-component', mode=InsideHTMLTag, absolute=14)])
+Line([Text('              hidden', mode=InsideHTMLTag, absolute=14), Text('/>', mode=DjHTML)], offset=14)
+Line([Text('<', mode=DjHTML), Text('my.component', mode=InsideHTMLTag, absolute=14)])
+Line([Text('              hidden', mode=InsideHTMLTag, absolute=14), Text('/>', mode=DjHTML)], offset=14)
+Line([Text('<', mode=DjHTML), Text('my:component', mode=InsideHTMLTag, absolute=14)])
+Line([Text('              hidden', mode=InsideHTMLTag, absolute=14), Text('/>', mode=DjHTML)], offset=14)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line attribute value with additional elements/attributes ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML), Text('text', mode=DjHTML), Text('<', mode=DjHTML), Text('blockquote ', mode=InsideHTMLTag, relative=-1, absolute=21), Text('id=', mode=InsideHTMLTag, relative=-1, absolute=21), Text('"', mode=InsideHTMLTag, relative=-1, absolute=25), Text('1', mode=InsideHTMLTag, relative=-1, absolute=25), Text('"', mode=InsideHTMLTag, relative=-1, absolute=24), Text(' ', mode=InsideHTMLTag, relative=-1, absolute=21), Text('id=', mode=InsideHTMLTag, relative=-1, absolute=21), Text('"', mode=InsideHTMLTag, relative=-1, absolute=32), Text('2', mode=InsideHTMLTag, relative=-1, absolute=32), Text('"', mode=InsideHTMLTag, relative=-1, absolute=31)])
+Line([Text('                     ', mode=InsideHTMLTag, relative=-1, absolute=21), Text('id=', mode=InsideHTMLTag, relative=-1, absolute=21), Text('"', mode=InsideHTMLTag, relative=-1, absolute=25), Text('3', mode=InsideHTMLTag, relative=-1, absolute=25), Text('"', mode=InsideHTMLTag, relative=-1, absolute=24), Text(' ', mode=InsideHTMLTag, relative=-1, absolute=21), Text('cite=', mode=InsideHTMLTag, relative=-1, absolute=21), Text('"', mode=InsideHTMLTag, relative=-1, absolute=34)], offset=21)
+Line([Text('                                  Wikipedia', mode=InsideHTMLTag, relative=-1, absolute=34)], offset=34)
+Line([Text('                                 ', mode=InsideHTMLTag, relative=-1, absolute=34), Text('"', mode=InsideHTMLTag, relative=-1, absolute=33)], offset=33)
+Line([Text('                     ', mode=InsideHTMLTag, relative=-1, absolute=21), Text('id=', mode=InsideHTMLTag, relative=-1, absolute=21), Text('"', mode=InsideHTMLTag, relative=-1, absolute=25), Text('4', mode=InsideHTMLTag, relative=-1, absolute=25), Text('"', mode=InsideHTMLTag, relative=-1, absolute=24), Text('/>', mode=DjHTML), Close('</div>', mode=DjHTML)], offset=21)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line elements with template tags ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8)])
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('{% include "attrs.html" %}', mode=DjTXT, absolute=3)], offset=3)
+Line([Text('   ', mode=InsideHTMLTag, absolute=3), Text('class=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=10)], offset=3)
+Line([Text('          ', mode=InsideHTMLTag, absolute=10), Open('{% block classes %}', mode=DjTXT, absolute=10)], offset=10)
+Line([Text('              foo', mode=InsideHTMLTag, absolute=10)], level=1, offset=10)
+Line([Text('              bar', mode=InsideHTMLTag, absolute=10)], level=1, offset=10)
+Line([Text('          ', mode=InsideHTMLTag, absolute=10), Close('{% endblock %}', mode=DjTXT, absolute=10)], offset=10)
+Line([Text('         ', mode=InsideHTMLTag, absolute=10), Text('"', mode=InsideHTMLTag, absolute=9)], offset=9)
+Line([Text('/>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The example from the README ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('blockquote ', mode=InsideHTMLTag, absolute=12), Text('cite=', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=18), Text('Guido Van Rossum', mode=InsideHTMLTag, absolute=18), Text('"', mode=InsideHTMLTag, absolute=17)])
+Line([Text('            ', mode=InsideHTMLTag, absolute=12), Text('style=', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=19), Text('font-style: italic;', mode=InsideHTMLTag, absolute=19)], offset=12)
+Line([Text('                   ', mode=InsideHTMLTag, absolute=19), Open('{% if dark_mode %}', mode=DjTXT, absolute=19)], offset=19)
+Line([Text('                       background: black;', mode=InsideHTMLTag, absolute=19)], level=1, offset=19)
+Line([Text('                   ', mode=InsideHTMLTag, absolute=19), Close('{% endif %}', mode=DjTXT, absolute=19)], offset=19)
+Line([Text('                  ', mode=InsideHTMLTag, absolute=19), Text('"', mode=InsideHTMLTag, absolute=18), Open('>', mode=DjHTML)], offset=18)
+Line([Text("    Don't you hate code that's not properly indented?", mode=DjHTML)], level=1)
+Line([Close('</blockquote>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Who knew HTML allowed single quotes? ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('blockquote ', mode=InsideHTMLTag, absolute=12), Text('class=', mode=InsideHTMLTag, absolute=12), Text("'", mode=InsideHTMLTag, absolute=19), Text('foo', mode=InsideHTMLTag, absolute=19)])
+Line([Text('                  ', mode=InsideHTMLTag, absolute=19), Text("'", mode=InsideHTMLTag, absolute=18)], offset=18)
+Line([Text('            ', mode=InsideHTMLTag, absolute=12), Text('class=', mode=InsideHTMLTag, absolute=12), Text("'", mode=InsideHTMLTag, absolute=19), Text('bar', mode=InsideHTMLTag, absolute=19), Text('"', mode=InsideHTMLTag, absolute=19)], offset=12)
+Line([Text('                  ', mode=InsideHTMLTag, absolute=19), Text("'", mode=InsideHTMLTag, absolute=18)], offset=18)
+Line([Text('            ', mode=InsideHTMLTag, absolute=12), Text('class=', mode=InsideHTMLTag, absolute=12), Text('"', mode=InsideHTMLTag, absolute=19), Text('baz', mode=InsideHTMLTag, absolute=19), Text("'", mode=InsideHTMLTag, absolute=19)], offset=12)
+Line([Text('                  ', mode=InsideHTMLTag, absolute=19), Text('"', mode=InsideHTMLTag, absolute=18)], offset=18)
+Line([Text('/>', mode=DjHTML)])
+Line([])
\ No newline at end of file
diff --git a/tests/suite/jinja.tokens b/tests/suite/jinja.tokens
new file mode 100644
index 0000000..744a4f2
--- /dev/null
+++ b/tests/suite/jinja.tokens
@@ -0,0 +1,55 @@
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multiline comment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{#', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('      ,-._|\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     /     .\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     \\_,--._/', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('#}', mode=DjTXT, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Raw block ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% raw %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([Text('      ,-._|\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     /     .\\', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Text('     \\_,--._/', mode=Comment, ignore=True)], level=1, ignore=True)
+Line([Close('{% endraw %}', mode=DjTXT, ignore=True)], ignore=True)
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Whitespace control ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Open('{%+ if something %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        yay', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Open('{% if something +%}', mode=DjTXT, level=1)], level=1)
+Line([Text('        yay', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([Open('{% for item in seq -%}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text('{{ item }}', mode=DjHTML)], level=1)
+Line([Close('{%- endfor %}', mode=DjTXT)])
+Line([])
+Line([Open('{%- macro render_currency(currency, amount) -%}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text('{{ amount }}', mode=DjHTML)], level=1)
+Line([Close('{%- endmacro -%}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Set assignment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('{% set "five" = 5 %}', mode=DjTXT)], level=1)
+Line([Text('    Five is ', mode=DjHTML), Text('{{ five }}', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Open('{% set blockdata %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        Contents', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endset %}', mode=DjTXT)], level=1)
+Line([Text('    Contents of block is block', mode=DjHTML)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(" Don't support multiline if statements (for now) ", mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('{% if ( condition1 and', mode=DjHTML)])
+Line([Text('condition2 ) %}', mode=DjHTML)])
+Line([Text('text', mode=DjHTML)])
+Line([Close('{% endif %}', mode=DjTXT)])
+Line([])
\ No newline at end of file
diff --git a/tests/suite/js.tokens b/tests/suite/js.tokens
new file mode 100644
index 0000000..d7a5f82
--- /dev/null
+++ b/tests/suite/js.tokens
@@ -0,0 +1,447 @@
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Basic Javascript ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    function f', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        return 42;', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Different brace types ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    array = ', mode=DjJS), Open('[', mode=DjJS, level=1)], level=1)
+Line([Text('        1,', mode=DjJS)], level=2)
+Line([Text('        2,', mode=DjJS)], level=2)
+Line([Text('        3,', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close(']', mode=DjJS)], level=1)
+Line([])
+Line([Text('    dictionary = ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        ', mode=DjJS), Text('value1:', mode=DjJS), Text(' 1,', mode=DjJS, relative=1)], level=2)
+Line([Text('        ', mode=DjJS), Text('value2:', mode=DjJS), Text(' 2,', mode=DjJS, relative=1)], level=2)
+Line([Text('        ', mode=DjJS), Text('value3:', mode=DjJS), Text(' 3,', mode=DjJS, relative=1)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([])
+Line([Text('    function_call', mode=DjJS), Open('(', mode=DjJS, level=1)], level=1)
+Line([Text('        arg1,', mode=DjJS)], level=2)
+Line([Text('        arg2,', mode=DjJS)], level=2)
+Line([Text('        arg3,', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Function chaining ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    one', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS)], level=1)
+Line([Text('        .two', mode=DjJS, relative=1), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS)], level=2)
+Line([Text('        .three', mode=DjJS, relative=1), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    four', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    function f', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        return 42;', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Double-nested braces ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    $', mode=DjJS), Open('(', mode=DjJS, level=1), Text('function', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        this.addEventListener', mode=DjJS), Open('(', mode=DjJS, level=2), Text("'change'", mode=DjJS), Text(', function ', mode=DjJS), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            doStuff', mode=DjJS), Open('(', mode=DjJS, level=3), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multiline comment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Open('/*', mode=DjJS, level=1, ignore=True)], level=1, ignore=True)
+Line([Text('     * Hi mom!', mode=Comment, ignore=True)], level=2, ignore=True)
+Line([Text('     *', mode=Comment, ignore=True)], level=2, ignore=True)
+Line([Text('     ', mode=Comment, ignore=True), Close('*/', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Nasty characters embedded in strings ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    console.log', mode=DjJS), Open('(', mode=DjJS, level=1), Text('"$(function() {"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Text('    console.log', mode=DjJS), Open('(', mode=DjJS, level=1), Text("'$(function() {'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Text('    console.log', mode=DjJS), Open('(', mode=DjJS, level=1), Text('`$(function() {`', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Text('    console.log', mode=DjJS), Open('(', mode=DjJS, level=1), Text('`\nfunction f() {\nreturn 42;\n}\n    `', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Real-life example kindly provided by @jacklinke ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('format_duration_value = ', mode=DjJS, absolute=6), Open('(', mode=DjJS, level=1), Text('a', mode=DjJS), Close(')', mode=DjJS), Text(' => ', mode=DjJS, relative=1, absolute=6), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        ', mode=DjJS), Text('let ', mode=DjJS), Text('secs = a * 3600;  ', mode=DjJS, absolute=4), Text('// get total amount of seconds from your input', mode=DjJS, ignore=True)], level=2)
+Line([Text('        ', mode=DjJS), Text('const ', mode=DjJS), Text('days = Math.floor', mode=DjJS, absolute=6), Open('(', mode=DjJS, level=2), Text('secs / ', mode=DjJS), Open('(', mode=DjJS, level=2), Text('3600 * 24', mode=DjJS), Close(')', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS, absolute=6)], level=2)
+Line([Text('        return ', mode=DjJS), Open('(', mode=DjJS, level=2), Text('`\n            <div class="form-inline text-nowrap text-center">\n                ${days.toString().padStart(2,\'0\')}\n                <small>m</small>\n            </div>\n        `', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Text(';', mode=DjJS, absolute=6)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' More Javascript shenanigans ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    $', mode=DjJS), Open('(', mode=DjJS, level=1), Text('function1', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS)], level=1)
+Line([Text('        ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            ', mode=DjJS), Text('var ', mode=DjJS), Text('foo = ', mode=DjJS, absolute=4), Open('(', mode=DjJS, level=3), Text('function2', mode=DjJS), Open('(', mode=DjJS, level=3), Close(')', mode=DjJS), Open('{', mode=DjJS, level=3)], level=3)
+Line([Text('                doStuff', mode=DjJS), Open('(', mode=DjJS, level=4), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=4)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Open('(', mode=DjJS, level=3), Close(')', mode=DjJS), Text(';', mode=DjJS, relative=1, absolute=4)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Text('// Jeez', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' If elif else ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('let ', mode=DjJS), Text('time = 10;', mode=DjJS, absolute=4)], level=1)
+Line([Text('    ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1), Text('time < 10', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        console.log', mode=DjJS), Open('(', mode=DjJS, level=2), Text('"Good morning"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Text(' else ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('time < 20', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        console.log', mode=DjJS), Open('(', mode=DjJS, level=2), Text('"Good day"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Text(' else ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        console.log', mode=DjJS), Open('(', mode=DjJS, level=2), Text('"Good evening"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Switch case indentation ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('mode = ', mode=DjJS, absolute=6), Text("'approved'", mode=DjJS, relative=1, absolute=6)], level=1)
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('mode', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'approved'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Approved'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'rejected'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Rejected'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Default'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([])
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('mode', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'approved'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Approved'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Default'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'rejected'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Rejected'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([])
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('mode', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'approved'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'rejected'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'invalid'", mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Not default'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Default'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([])
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('mode', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'approved'", mode=DjJS), Text(': ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Approved'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS)], level=2)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text("'rejected'", mode=DjJS), Text(': ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Rejected'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS)], level=2)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=2), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            console.log', mode=DjJS), Open('(', mode=DjJS, level=3), Text("'Default'", mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Switch complex example ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('year = 2016;', mode=DjJS, absolute=4)], level=1)
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('month = 2;', mode=DjJS, absolute=4)], level=1)
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('dayCount;', mode=DjJS, absolute=4)], level=1)
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('month', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('1:', mode=DjJS)], level=2)
+Line([Text('            function switchFoo ', mode=DjJS), Open('(', mode=DjJS, level=3), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=3)], level=3)
+Line([Text('                console.log', mode=DjJS), Open('(', mode=DjJS, level=4), Text('"Foo"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=4)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS)], level=3)
+Line([Text('            switchFoo', mode=DjJS), Open('(', mode=DjJS, level=3), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=3)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('12:', mode=DjJS)], level=2)
+Line([Text('            dayCount = 31;', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('4:', mode=DjJS)], level=2)
+Line([Text('            switch', mode=DjJS), Open('(', mode=DjJS, level=3), Text('boo', mode=DjJS), Close(')', mode=DjJS), Open('{', mode=DjJS, level=3)], level=3)
+Line([OpenDouble('                case ', mode=DjJS, level=4), Text('1:', mode=DjJS)], level=4)
+Line([OpenDouble('                case ', mode=DjJS, level=4), Text('2:', mode=DjJS)], level=4)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS)], level=3)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('11:', mode=DjJS)], level=2)
+Line([Text('            dayCount = 30;', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('2:', mode=DjJS)], level=2)
+Line([Text('            ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=3), Open('(', mode=DjJS, level=3), Open('(', mode=DjJS, level=3), Text('year % 4 == 0', mode=DjJS), Close(')', mode=DjJS), Text(' && !', mode=DjJS), Open('(', mode=DjJS, level=3), Text('year % 100 == 0', mode=DjJS), Close(')', mode=DjJS), Close(')', mode=DjJS), Text(' || ', mode=DjJS), Open('(', mode=DjJS, level=3), Text('year % 400 == 0', mode=DjJS), Close(')', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('{', mode=DjJS, level=3)], level=3)
+Line([Text('                dayCount = 29;', mode=DjJS)], level=4)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS), Text(' else ', mode=DjJS), Open('{', mode=DjJS, level=3)], level=3)
+Line([Text('                dayCount = 28;', mode=DjJS)], level=4)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS)], level=3)
+Line([Text('            break;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=2)], level=2)
+Line([Text('            dayCount = -1; ', mode=DjJS), Text('// invalid month', mode=DjJS, ignore=True)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Text('    console.log', mode=DjJS), Open('(', mode=DjJS, level=1), Text('dayCount', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Nested switch statements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('a', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('b:', mode=DjJS)], level=2)
+Line([Text('            switch ', mode=DjJS), Open('(', mode=DjJS, level=3), Text('c', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=3)], level=3)
+Line([OpenDouble('                case ', mode=DjJS, level=4), Text('d:', mode=DjJS)], level=4)
+Line([Text('                    switch ', mode=DjJS), Open('(', mode=DjJS, level=5), Text('e', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=5)], level=5)
+Line([Text('                        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=6)], level=6)
+Line([Text('                    ', mode=DjJS), Close('}', mode=DjJS)], level=5)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Succinct switch statement ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('foo', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('bar:', mode=DjJS), Text(' break', mode=DjJS)], level=2)
+Line([Text('        ', mode=DjJS), OpenDouble('default:', mode=DjJS, level=2), Text(' return', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(" Things that look like switch, but aren't ", mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    function switchBoo ', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        console.log', mode=DjJS), Open('(', mode=DjJS, level=2), Text('"Boo"', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Text('    switchBoo', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([])
+Line([Text('    ', mode=DjJS), Text('let ', mode=DjJS), Text('arr = ', mode=DjJS, absolute=4), Open('[', mode=DjJS, level=1)], level=1)
+Line([Text('        ', mode=DjJS), Text("'switch'", mode=DjJS), Text(',', mode=DjJS)], level=2)
+Line([Text('        ', mode=DjJS), Text("'default:'", mode=DjJS), Text(',', mode=DjJS)], level=2)
+Line([Text('        ', mode=DjJS), Text("'case '", mode=DjJS), Text(',', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close(']', mode=DjJS), Text(';', mode=DjJS, absolute=4)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Another problematic switch statement ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    switch ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('foo', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=1)], level=1)
+Line([OpenDouble('        case ', mode=DjJS, level=2), Text('bar:', mode=DjJS)], level=2)
+Line([Text('            ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=3), Text('baz', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('{', mode=DjJS, level=3)], level=3)
+Line([Text('                switch ', mode=DjJS), Open('(', mode=DjJS, level=4), Text('foo', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=4)], level=4)
+Line([OpenDouble('                    case ', mode=DjJS, level=5), Text('bar:', mode=DjJS)], level=5)
+Line([Text('                        ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=6), Text('baz', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('{', mode=DjJS, level=6)], level=6)
+Line([Text('                            xizzy', mode=DjJS), Open('(', mode=DjJS, level=7), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=7)
+Line([Text('                        ', mode=DjJS), Close('}', mode=DjJS)], level=6)
+Line([Text('                ', mode=DjJS), Close('}', mode=DjJS)], level=4)
+Line([Text('            ', mode=DjJS), Close('}', mode=DjJS)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' More function chaining ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    window.fetch', mode=DjJS), Open('(', mode=DjJS, level=1), Text("'/test.html'", mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('        .then', mode=DjJS, relative=1), Open('(', mode=DjJS, level=2), Open('(', mode=DjJS, level=2), Text('html', mode=DjJS), Close(')', mode=DjJS), Text(' => ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            document.body.innerHTML = html;', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([])
+Line([Text('    foo', mode=DjJS)], level=1)
+Line([Text('        .bar', mode=DjJS, relative=1)], level=2)
+Line([Text('        .baz', mode=DjJS, relative=1)], level=2)
+Line([Text('        .then', mode=DjJS, relative=1), Open('(', mode=DjJS, level=2), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(' => ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            foo', mode=DjJS)], level=3)
+Line([Text('                .bar', mode=DjJS, relative=1)], level=4)
+Line([Text('                .baz', mode=DjJS, relative=1)], level=4)
+Line([Text('                .then', mode=DjJS, relative=1), Open('(', mode=DjJS, level=4), Open('(', mode=DjJS, level=4), Close(')', mode=DjJS), Text(' => ', mode=DjJS), Open('{', mode=DjJS, level=4)], level=4)
+Line([Text('                    foo', mode=DjJS)], level=5)
+Line([Text('                        .bar', mode=DjJS, relative=1)], level=6)
+Line([Text('                        .baz', mode=DjJS, relative=1)], level=6)
+Line([Text('                ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=4)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=2)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Not enough closing tokens ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('head', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        function foo', mode=DjJS), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('    ', mode=DjJS), Close('</script>', mode=DjHTML)], level=1)
+Line([Close('</head>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Too much closing tokens ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('head', mode=InsideHTMLTag, absolute=6), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        function foo', mode=DjJS), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS)], level=2)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS)], level=2)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('</script>', mode=DjHTML)], level=1)
+Line([Close('</head>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Single-line comments ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('// The following does not contain a comment', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    wsHost = ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('/^wss?:\\/\\//', mode=DjJS), Text('i.test', mode=DjJS), Open('(', mode=DjJS, level=1), Text('me._host', mode=DjJS), Close(')', mode=DjJS), Text(' ? ', mode=DjJS), Text("''", mode=DjJS), Text(' : me._protocol', mode=DjJS), Close(')', mode=DjJS), Text(' + me._host;', mode=DjJS, relative=1)], level=1)
+Line([])
+Line([Text('    foo; ', mode=DjJS), Text('// This is a comment', mode=DjJS, ignore=True)], level=1)
+Line([Text('    bar ', mode=DjJS), Text('//This too', mode=DjJS, ignore=True)], level=1)
+Line([Text('    baz', mode=DjJS), Text('//Even this', mode=DjJS, ignore=True)], level=1)
+Line([])
+Line([Text('    foo ', mode=DjJS), Text('// function() {', mode=DjJS, ignore=True)], level=1)
+Line([Text('    bar ', mode=DjJS), Text('//     var a = 1;', mode=DjJS, ignore=True)], level=1)
+Line([Text('    baz ', mode=DjJS), Text('// }', mode=DjJS, ignore=True)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line variable assignment ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('a = 1,', mode=DjJS, absolute=4)], level=1)
+Line([Text('        b = 2,', mode=DjJS, absolute=4)], level=1, offset=4)
+Line([Text('        c;', mode=DjJS, absolute=4)], level=1, offset=4)
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('foo = bar; ', mode=DjJS, absolute=4), Text('let ', mode=DjJS), Text('a = 1,', mode=DjJS, absolute=19)], level=1)
+Line([Text('                       b = 2,', mode=DjJS, absolute=19)], level=1, offset=19)
+Line([Text('                       c', mode=DjJS, absolute=19)], level=1, offset=19)
+Line([])
+Line([Text('    ', mode=DjJS), Text('// Haskell-style indentation', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('a = 1', mode=DjJS, absolute=6)], level=1)
+Line([Text('        , b = 2', mode=DjJS, absolute=4)], level=1, offset=4)
+Line([Text('        , c', mode=DjJS, absolute=4)], level=1, offset=4)
+Line([])
+Line([Text('    ', mode=DjJS), Text('// Notice the first line is the same', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('a = 1', mode=DjJS, absolute=6)], level=1)
+Line([Text('    b = 2', mode=DjJS)], level=1)
+Line([Text('    c = 3', mode=DjJS)], level=1)
+Line([])
+Line([Text('    ', mode=DjJS), Text('// Yes, this makes sense', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('func1 = function', mode=DjJS, absolute=6), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS, absolute=6), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        return foo', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Text(',', mode=DjJS, absolute=6)], level=1)
+Line([Text('          func2 = function', mode=DjJS, absolute=6), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS, absolute=6), Open('{', mode=DjJS, level=1), Text(' return 2 ', mode=DjJS), Close('}', mode=DjJS), Text(',', mode=DjJS, absolute=6)], level=1, offset=6)
+Line([Text('          func3 = function', mode=DjJS, absolute=6), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(' ', mode=DjJS, absolute=6), Open('{', mode=DjJS, level=1), Text(' return 3 ', mode=DjJS), Close('}', mode=DjJS)], level=1, offset=6)
+Line([])
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('foo = bar,', mode=DjJS, absolute=6)], level=1)
+Line([Text('          baz = ', mode=DjJS, absolute=6), Open('{', mode=DjJS, level=1)], level=1, offset=6)
+Line([Text('        ...', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([])
+Line([Text('    ', mode=DjJS), Text('// We are not supporting madness like this', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('a = 1,', mode=DjJS, absolute=6)], level=1)
+Line([Text('    ', mode=DjJS), Open('{% if %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        b = 2,', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Text('    c = 3;', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Things that look like case statements, and the "spread" operator ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('case = ', mode=DjJS, absolute=4), Text('"foo"', mode=DjJS, relative=1, absolute=4)], level=1)
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('dict = ', mode=DjJS, absolute=4), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        ', mode=DjJS), Text('case:', mode=DjJS), Text(' ', mode=DjJS, relative=1), Text('"bar"', mode=DjJS, relative=1), Text(',', mode=DjJS)], level=2)
+Line([Text('        ', mode=DjJS), Text('"default:"', mode=DjJS), Text(': ', mode=DjJS), Text('"baz"', mode=DjJS, relative=1), Text(',', mode=DjJS)], level=2)
+Line([Text('        ...params,', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line ternary operator ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    foo', mode=DjJS)], level=1)
+Line([Text('        ? bar', mode=DjJS, relative=1)], level=2)
+Line([Text('        : baz', mode=DjJS, relative=1)], level=2)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Quoted strings with escaped quotes ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('foo = ', mode=DjJS, absolute=4), Text("'\\'{'", mode=DjJS, relative=1, absolute=4)], level=1)
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('bar = ', mode=DjJS, absolute=4), Text('"\\"{"', mode=DjJS, relative=1, absolute=4)], level=1)
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('baz = ', mode=DjJS, absolute=4), Text('`\\`{`', mode=DjJS, relative=1, absolute=4)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Single-line statements ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1), Text('foo', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('        bar', mode=DjJS, relative=1)], level=2)
+Line([])
+Line([Text('    ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1), Text('foo', mode=DjJS), Close(')', mode=DjJS), Text(' return', mode=DjJS, relative=1)], level=1)
+Line([Text('    bar', mode=DjJS)], level=1)
+Line([])
+Line([Text('    ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1)], level=1)
+Line([Text('        foo &&', mode=DjJS)], level=2)
+Line([Text('        bar', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('        baz', mode=DjJS, relative=1)], level=2)
+Line([Text('    ', mode=DjJS), Text('else', mode=DjJS)], level=1)
+Line([Text('        return', mode=DjJS, relative=1)], level=2)
+Line([])
+Line([Text('    ', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1), Text('foo', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('        ', mode=DjJS, relative=1), Text('if', mode=DjJS, relative=1), Text(' ', mode=DjJS, relative=2), Open('(', mode=DjJS, level=2), Text('bar', mode=DjJS), Close(')', mode=DjJS)], level=2)
+Line([Text('            ', mode=DjJS, relative=2), Text('if', mode=DjJS, relative=2), Text(' ', mode=DjJS, relative=3), Open('(', mode=DjJS, level=3), Text('baz', mode=DjJS), Close(')', mode=DjJS)], level=3)
+Line([Text('                return', mode=DjJS, relative=3)], level=4)
+Line([Text('    ', mode=DjJS), Text('else', mode=DjJS)], level=1)
+Line([Text('        console.log', mode=DjJS, relative=1), Open('(', mode=DjJS, level=2), Text('"Footgun detected!"', mode=DjJS), Close(')', mode=DjJS)], level=2)
+Line([])
+Line([Text('    leitmot', mode=DjJS), Text('if', mode=DjJS), Text(' ', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS)], level=1)
+Line([Text('    elsewhere ', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Text('    mean', mode=DjJS), Text('while', mode=DjJS), Text(' ', mode=DjJS), Open('(', mode=DjJS, level=1), Text('true', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([])
+Line([Text('    ', mode=DjJS), Text('while', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1), Text('true', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('        break', mode=DjJS, relative=1)], level=2)
+Line([])
+Line([Text('    ', mode=DjJS), Text('for', mode=DjJS), Text(' ', mode=DjJS, relative=1), Open('(', mode=DjJS, level=1), Text('i=1, 1<10, i++', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('    ', mode=DjJS, relative=1), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        break', mode=DjJS)], level=2)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Multi-line variables and dictionaries ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Text('var ', mode=DjJS), Text('foo =', mode=DjJS, absolute=4)], level=1)
+Line([Text('        bar', mode=DjJS, relative=1)], level=2)
+Line([])
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('func =', mode=DjJS, absolute=6)], level=1)
+Line([Text('        function', mode=DjJS, relative=1), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('            return', mode=DjJS)], level=3)
+Line([Text('        ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS)], level=2)
+Line([])
+Line([Text('    ', mode=DjJS), Text('// Note that unquoted "default" is a reserved keyword', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=DjJS), Text('let ', mode=DjJS), Text('dict = ', mode=DjJS, absolute=4), Open('{', mode=DjJS, level=1)], level=1)
+Line([Text('        ', mode=DjJS), Text('"default"', mode=DjJS), Text(':', mode=DjJS)], level=2)
+Line([Text('            foo', mode=DjJS, relative=1)], level=3)
+Line([Text('        ', mode=DjJS), Text('bar:', mode=DjJS)], level=2)
+Line([Text('            baz', mode=DjJS, relative=1)], level=3)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Expression literals ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    re = ', mode=DjJS), Text('/ab+c/', mode=DjJS, relative=1)], level=1)
+Line([Text('    space_re = ', mode=DjJS), Text('/ foo /', mode=DjJS, relative=1)], level=1)
+Line([Text('    single_re = ', mode=DjJS), Text('/foo\\/ + bar/', mode=DjJS, relative=1)], level=1)
+Line([Text('    double_re = ', mode=DjJS), Text('/foo/', mode=DjJS, relative=1), Text(' + ', mode=DjJS), Text('/bar/', mode=DjJS)], level=1)
+Line([Text('    look_like_comment = ', mode=DjJS), Text('/\\(/', mode=DjJS, relative=1), Text('/', mode=DjJS)], level=1)
+Line([])
+Line([Text('    ', mode=DjJS), Text('// The following is known to be broken', mode=DjJS, ignore=True)], level=1, ignore=True)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('x = 42', mode=DjJS, absolute=6)], level=1)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('y = 7', mode=DjJS, absolute=6)], level=1)
+Line([Text('    ', mode=DjJS), Text('const ', mode=DjJS), Text('six = x ', mode=DjJS, absolute=6), Text('/ (function() { /', mode=DjJS, absolute=6), Text('/ comment', mode=DjJS, absolute=6)], level=1)
+Line([Text('    return y', mode=DjJS)], level=1)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Open('(', mode=DjJS, level=1), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
\ No newline at end of file
diff --git a/tests/suite/paradoxes.tokens b/tests/suite/paradoxes.tokens
new file mode 100644
index 0000000..3fbcfab
--- /dev/null
+++ b/tests/suite/paradoxes.tokens
@@ -0,0 +1,51 @@
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Common pattern in django templates ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('div', mode=InsideHTMLTag, absolute=5), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjHTML), Open('{% if some_condition %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('{% url a %}', mode=DjTXT, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('    ', mode=DjHTML), CloseAndOpen('{% elif some_other condition %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('{% url b %}', mode=DjTXT, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Text('    .', mode=DjHTML)], level=1)
+Line([Text('    ', mode=DjHTML), Open('{% if some_condition or some_other_condition %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Close('</a>', mode=DjHTML)], level=2)
+Line([Text('    ', mode=DjHTML), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Close('</div>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' The literal example from issue #17 ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% if some_condition %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('{% url some_url', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=1)], level=1)
+Line([CloseAndOpen('{% elif some_other_condition %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('a ', mode=InsideHTMLTag, absolute=3), Text('href=', mode=InsideHTMLTag, absolute=3), Text('"', mode=InsideHTMLTag, absolute=9), Text('{% url some_other_url', mode=InsideHTMLTag, absolute=9), Text('"', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Close('{% endif %}', mode=DjTXT)])
+Line([Text('[a bunch of details go here]', mode=DjHTML)])
+Line([Open('{% if end_tag_needed %}', mode=DjTXT), Close('</a>', mode=DjHTML), Close('{% endif %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Deep nestings should be instantly dedented ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Open('{% block %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3), Open('>', mode=DjHTML, level=1)], level=1)
+Line([Text('        ', mode=DjHTML), Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3), Open('>', mode=DjHTML, level=2)], level=2)
+Line([Text('            ', mode=DjHTML), Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3), Open('>', mode=DjHTML, level=3)], level=3)
+Line([Text('                ', mode=DjHTML), Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3), Open('>', mode=DjHTML, level=4)], level=4)
+Line([Text('                    ', mode=DjHTML), Text('<', mode=DjHTML), Text('a', mode=InsideHTMLTag, absolute=3), Open('>', mode=DjHTML, level=5)], level=5)
+Line([Close('{% endblock %}', mode=DjTXT)])
+Line([Open('{% for x in [1,2,3,4,5] %}', mode=DjTXT)])
+Line([Text('    ', mode=DjHTML), Close('</a>', mode=DjHTML)], level=1)
+Line([Close('{% endfor %}', mode=DjTXT)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Inside <script> tag ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([Text('<', mode=DjHTML), Text('script', mode=InsideHTMLTag, absolute=8), Open('>', mode=DjHTML)])
+Line([Text('    ', mode=DjJS), Open('{% if jquery %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        $', mode=DjJS), Open('(', mode=DjJS, level=2), Text('function', mode=DjJS), Open('(', mode=DjJS, level=2), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('    ', mode=DjJS), CloseAndOpen('{% else %}', mode=DjTXT, level=1)], level=1)
+Line([Text('        document.addEventListener', mode=DjJS), Open('(', mode=DjJS, level=2), Text('"DOMContentLoaded"', mode=DjJS), Text(', function', mode=DjJS), Open('(', mode=DjJS, level=2), Text('event', mode=DjJS), Close(')', mode=DjJS), Text(' ', mode=DjJS), Open('{', mode=DjJS, level=2)], level=2)
+Line([Text('    ', mode=DjJS), Close('{% endif %}', mode=DjTXT)], level=1)
+Line([Text('    console.log', mode=DjJS), Open('(', mode=DjJS, level=1), Text('"Time to ditch jQuery!"', mode=DjJS), Close(')', mode=DjJS)], level=1)
+Line([Text('    ', mode=DjJS), Close('}', mode=DjJS), Close(')', mode=DjJS), Text(';', mode=DjJS)], level=1)
+Line([Close('</script>', mode=DjHTML)])
+Line([])
+Line([])
+Line([Open('<!--', mode=DjHTML, ignore=True), Text(' Final note: all other malstructured source files will result in undefined behavior, but not in errors ', mode=Comment, ignore=True), Close('-->', mode=DjHTML, ignore=True)], ignore=True)
+Line([])
\ No newline at end of file
diff --git a/tests/test_suite.py b/tests/test_suite.py
index d1979e2..e7a5c1e 100644
--- a/tests/test_suite.py
+++ b/tests/test_suite.py
@@ -1,12 +1,12 @@
-import os
 import unittest
+from pathlib import Path
 
 from djhtml.modes import DjHTML
 
 
 class TestSuite(unittest.TestCase):
     maxDiff = None
-    DIR = os.path.join(os.path.dirname(__file__), "suite")
+    DIR = Path(__file__).parent / "suite"
 
     def test_available_files(self):
         """
@@ -14,14 +14,18 @@ def test_available_files(self):
         expected output to the actual output.
 
         """
-        for filename in os.listdir(self.DIR):
-            with self.subTest(filename):
-                self._test_file(filename)
+        for filename in self.DIR.iterdir():
+            if filename.suffix == ".html":
+                with self.subTest(filename):
+                    self._test_file(filename.stem)
 
-    def _test_file(self, filename):
-        with open(os.path.join(self.DIR, filename)) as f:
+    def _test_file(self, basename):
+        with open(self.DIR / (basename + ".html")) as f:
             expected_output = f.read()
 
+        with open(self.DIR / (basename + ".tokens")) as f:
+            expected_tokens = f.read()
+
         # Indent the expected output to 0 (no indentation)
         unindented = DjHTML(expected_output).indent(0)
         self.assertNotEqual(unindented, expected_output)
@@ -29,3 +33,7 @@ def _test_file(self, filename):
         # Re-indent the unindented output to 4
         actual_output = DjHTML(unindented).indent(4)
         self.assertEqual(expected_output, actual_output)
+
+        # Compare the tokenization
+        actual_tokens = DjHTML(actual_output).debug()
+        self.assertEqual(expected_tokens, actual_tokens)