Fix read chunks (#291)

* Clean read chunked response

* Remove walrus operator for python 3.7

* Simplify logic for put request

* Fix tests

* Deprecated field accept_put_without_content_length

* Update CHANGELOG.md

---------

Co-authored-by: Martin Wendt <github@wwwendt.de>

CHANGELOG.md

@@ -7,7 +7,7 @@
 - Deprecate Python 3.7 (EOL: 2023-06-27)
 - Install pam_dc dependencies using extra syntax: `pip install wsgidav[pam]`
 - #281 Requesting range off end of file does not return 416 status code
-- #282 Hotfix PUT request without content-length (fix for Finder on MacOS Ventura)
+- #290 Unable to upload chunked big file using requests python library.
 - Add `logging.enable` option to activate the 'wsgidav' logger when this package
   is used as a library. This replaces an explicit call to `utils.init_logging()`.
   When running as CLI, this option is on by default.

sample_wsgidav.yaml

@@ -43,10 +43,6 @@ block_size: 8192
 add_header_MS_Author_Via: true
 
 hotfixes:
-    #: Some clients send PUT requests without a body and omit the 
-    #: `Content-Length` header ("Microsoft-WebDAV-MiniRedir", "gvfs/", "Darwin").
-    #: See issues #10, #282. This option assumes "0" for missing headers:
-    accept_put_without_content_length: true
     #: Handle Microsoft's Win32LastModifiedTime property.
     #: This is useful only in the case when you copy files from a Windows
     #: client into a WebDAV share. Windows sends the "last modified" time of

wsgidav/default_conf.py

@@ -44,7 +44,7 @@
         "emulate_win32_lastmod": False,  # True: support Win32LastModifiedTime
         "re_encode_path_info": True,  # (See issue #73)
         "unquote_path_info": False,  # (See issue #8, #228)
-        "accept_put_without_content_length": True,  # (See issue #10, #282)
+        # "accept_put_without_content_length": True,  # (See issue #10, #282)
         # "treat_root_options_as_asterisk": False, # Hotfix for WinXP / Vista: accept 'OPTIONS /' for a 'OPTIONS *'
         # "win_accept_anonymous_options": False,
         # "winxp_accept_root_share_login": False,

wsgidav/fs_dav_provider.py

@@ -462,7 +462,7 @@ def get_resource_inst(self, path: str, environ: dict) -> FileResource:
         """
         self._count_get_resource_inst += 1
         fp = self._loc_to_file_path(path, environ)
-        print(f"resolve {path} => {fp}")
+
         if not os.path.exists(fp):
             return None
         if not self.fs_opts.get("follow_symlinks") and os.path.islink(fp):

wsgidav/request_server.py

@@ -17,7 +17,6 @@
     HTTP_FAILED_DEPENDENCY,
     HTTP_FORBIDDEN,
     HTTP_INTERNAL_ERROR,
-    HTTP_LENGTH_REQUIRED,
     HTTP_MEDIATYPE_NOT_SUPPORTED,
     HTTP_METHOD_NOT_ALLOWED,
     HTTP_NO_CONTENT,
@@ -662,93 +661,16 @@ def do_DELETE(self, environ, start_response):
             environ, start_response, res, HTTP_NO_CONTENT, error_list
         )
 
-    def _stream_data_chunked(self, environ, block_size):
-        """Get the data from a chunked transfer."""
-        # Chunked Transfer Coding
-        # http://www.servlets.com/rfcs/rfc2616-sec3.html#sec3.6.1
-
-        if "Darwin" in environ.get("HTTP_USER_AGENT", "") and environ.get(
-            "HTTP_X_EXPECTED_ENTITY_LENGTH"
-        ):
-            # Mac Finder, that does not prepend chunk-size + CRLF ,
-            # like it should to comply with the spec. It sends chunk
-            # size as integer in a HTTP header instead.
-            WORKAROUND_CHUNK_LENGTH = True
-            buf = environ.get("HTTP_X_EXPECTED_ENTITY_LENGTH", "0")
-            length = int(buf)
-        else:
-            WORKAROUND_CHUNK_LENGTH = False
-            buf = environ["wsgi.input"].readline()
-            environ["wsgidav.some_input_read"] = 1
-            if buf == b"":
-                length = 0
-            else:
-                length = int(buf, 16)
-
-        while length > 0:
+    def _stream_data(self, environ, block_size):
+        """Get the data."""
+        while True:
             buf = environ["wsgi.input"].read(block_size)
+            if buf == b"":
+                break
+            environ["wsgidav.some_input_read"] = 1
             yield buf
-            if WORKAROUND_CHUNK_LENGTH:
-                environ["wsgidav.some_input_read"] = 1
-                # Keep receiving until we read expected size or reach
-                # EOF
-                if buf == b"":
-                    length = 0
-                else:
-                    length -= len(buf)
-            else:
-                environ["wsgi.input"].readline()
-                buf = environ["wsgi.input"].readline()
-                if buf == b"":
-                    length = 0
-                else:
-                    length = int(buf, 16)
         environ["wsgidav.all_input_read"] = 1
 
-    def _stream_data(self, environ, content_length, block_size):
-        """Get the data from a non-chunked transfer."""
-        if content_length == 0:
-            # TODO: review this
-            # Windows MiniRedir submit PUT with Content-Length 0,
-            # before LOCK and the real PUT. So we have to accept this.
-            _logger.debug("PUT: Content-Length == 0. Creating empty file...")
-
-        #        elif content_length < 0:
-        #            # TODO: review this
-        #            # If CONTENT_LENGTH is invalid, we may try to workaround this
-        #            # by reading until the end of the stream. This may block however!
-        #            # The iterator produced small chunks of varying size, but not
-        #            # sure, if we always get everything before it times out.
-        #            _logger.warning("PUT with invalid Content-Length (%s). "
-        #                            "Trying to read all (this may timeout)..."
-        #                            .format(environ.get("CONTENT_LENGTH")))
-        #            nb = 0
-        #            try:
-        #                for s in environ["wsgi.input"]:
-        #                    environ["wsgidav.some_input_read"] = 1
-        #                    _logger.debug("PUT: read from wsgi.input.__iter__, len=%s" % len(s))
-        #                    yield s
-        #                    nb += len (s)
-        #            except socket.timeout:
-        #                _logger.warning("PUT: input timed out after writing %s bytes" % nb)
-        #                hasErrors = True
-        else:
-            assert content_length > 0
-            contentremain = content_length
-            while contentremain > 0:
-                n = min(contentremain, block_size)
-                readbuffer = environ["wsgi.input"].read(n)
-                # This happens with litmus expect-100 test:
-                if not len(readbuffer) > 0:
-                    _logger.error("input.read({}) returned 0 bytes".format(n))
-                    break
-                environ["wsgidav.some_input_read"] = 1
-                yield readbuffer
-                contentremain -= len(readbuffer)
-
-            if contentremain == 0:
-                environ["wsgidav.all_input_read"] = 1
-
     def do_PUT(self, environ, start_response):
         """
         @see: http://www.webdav.org/specs/rfc4918.html#METHOD_PUT
@@ -788,48 +710,9 @@ def do_PUT(self, environ, start_response):
         else:
             self._check_write_permission(res, "0", environ)
 
-        # Start Content Processing
-        # Content-Length may be 0 or greater. (Set to -1 if missing or invalid.)
-        try:
-            content_length = max(-1, int(environ.get("CONTENT_LENGTH", -1)))
-        except ValueError:
-            content_length = -1
-
-        if (content_length < 0) and (
-            environ.get("HTTP_TRANSFER_ENCODING", "").lower() != "chunked"
-        ):
-            # HOTFIX: not fully understood, but MS sends PUT without content-length,
-            # when creating new files
-
-            config = environ["wsgidav.config"]
-            hotfixes = util.get_dict_value(config, "hotfixes", as_dict=True)
-            accept_put_without_content_length = hotfixes.get(
-                "accept_put_without_content_length", True
-            )
-
-            # if ( "Microsoft-WebDAV-MiniRedir" in agent or "gvfs/" in agent):
-            if accept_put_without_content_length:  # issue #10, #282
-                agent = environ.get("HTTP_USER_AGENT", "")
-                _logger.warning(
-                    f"Set misssing Content-Length to 0 for PUT, agent={agent!r}"
-                )
-                content_length = 0
-            else:
-                util.fail(
-                    HTTP_LENGTH_REQUIRED,
-                    "PUT request with invalid Content-Length: ({})".format(
-                        environ.get("CONTENT_LENGTH")
-                    ),
-                )
-
         hasErrors = False
         try:
-            if environ.get("HTTP_TRANSFER_ENCODING", "").lower() == "chunked":
-                data_stream = self._stream_data_chunked(environ, self.block_size)
-            else:
-                data_stream = self._stream_data(
-                    environ, content_length, self.block_size
-                )
+            data_stream = self._stream_data(environ, self.block_size)
 
             fileobj = res.begin_write(content_type=environ.get("CONTENT_TYPE"))
 

wsgidav/wsgidav_app.py

@@ -86,6 +86,7 @@ def _check_config(config):
     deprecated_fields = {
         "acceptbasic": "http_authenticator.accept_basic",
         "acceptdigest": "http_authenticator.accept_digest",
+        "accept_put_without_content_length": "(removed)",
         "catchall": "error_printer.catch_all",
         "debug_litmus": "logging.debug_litmus",
         "debug_methods": "logging.debug_methods",