Introduced new chunked transfer encoding parser to remove chunk marke…

…rs in streaming clients.
cpp-netlib · deanberris · Apr 3, 2018 · Dec 14, 2016 · Dec 14, 2016 · Dec 15, 2016
commit 16f5c9d7da3d57c5a5f1434000ec541b1efec2f2
diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp
@@ -42,14 +42,15 @@ struct async_client
 
   async_client(bool cache_resolved, bool follow_redirect,
                bool always_verify_peer, int timeout,
+               bool remove_chunk_markers,
                std::shared_ptr<boost::asio::io_service> service,
                optional<string_type> certificate_filename,
                optional<string_type> verify_path,
                optional<string_type> certificate_file,
                optional<string_type> private_key_file,
                optional<string_type> ciphers,
                optional<string_type> sni_hostname, long ssl_options)
-      : connection_base(cache_resolved, follow_redirect, timeout),
+      : connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers),
         service_ptr(service.get() ? service
                                   : std::make_shared<boost::asio::io_service>()),
         service_(*service_ptr),

diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp
@@ -43,7 +43,7 @@ struct async_connection_base {
   // tag.
   static connection_ptr new_connection(
       resolve_function resolve, resolver_type &resolver, bool follow_redirect,
-      bool always_verify_peer, bool https, int timeout,
+      bool always_verify_peer, bool https, int timeout, bool remove_chunk_markers,
       optional<string_type> certificate_filename = optional<string_type>(),
       optional<string_type> const &verify_path = optional<string_type>(),
       optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +59,7 @@ struct async_connection_base {
         certificate_filename, verify_path, certificate_file, private_key_file,
         ciphers, sni_hostname, ssl_options);
     auto temp = std::make_shared<async_connection>(
-        resolver, resolve, follow_redirect, timeout, std::move(delegate));
+        resolver, resolve, follow_redirect, timeout, remove_chunk_markers, std::move(delegate));
     BOOST_ASSERT(temp != nullptr);
     return temp;
   }

diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -11,6 +11,7 @@
 
 #include <iterator>
 #include <cstdint>
+#include <iostream>
 #include <boost/algorithm/string/trim.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <boost/asio/placeholders.hpp>
@@ -37,6 +38,77 @@ namespace network {
 namespace http {
 namespace impl {
 
+template <class Tag>
+struct chunk_encoding_parser {
+
+  chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
+
+  enum state_t { header, header_end, data, data_end };
+
+  state_t state;
+  size_t chunk_size;
+  std::array<typename char_<Tag>::type, 1024> buffer;
+
+  void update_chunk_size(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
+    if (range.empty())
+        return;
+    std::stringstream ss;
+    ss << std::hex << range;
+    size_t size;
+    ss >> size;
+    chunk_size = (chunk_size << (range.size()*4)) + size;
+  }
+
+  boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> operator()(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
+    auto iter = boost::begin(range);
+    auto begin = iter;
+    auto pos = boost::begin(buffer);
+
+    while (iter != boost::end(range))
+      switch(state) {
+        case state_t::header:
+          iter = std::find(iter, boost::end(range), '\r');
+          update_chunk_size(boost::make_iterator_range(begin, iter));
+          if (iter != boost::end(range)) {
+            state = state_t::header_end;
+            ++iter;
+          }
+          break;
+
+        case state_t::header_end:
+          BOOST_ASSERT(*iter == '\n');
+          ++iter;
+          state = state_t::data;
+          break;
+
+        case state_t::data:
+          if (chunk_size == 0) {
+            BOOST_ASSERT(*iter == '\r');
+            ++iter;
+            state = state_t::data_end;
+          } else {
+            auto len = std::min(chunk_size, (size_t)std::distance(iter, boost::end(range)));
+            begin = iter;
+            iter = std::next(iter, len);
+            pos = std::copy(begin, iter, pos);
+            chunk_size -= len;
+          }
+          break;
+
+        case state_t::data_end:
+          BOOST_ASSERT (*iter == '\n');
+          ++iter;
+          begin = iter;
+          state = state_t::header;
+          break;
+
+        default:
+          BOOST_ASSERT(false && "Bug, report this to the developers!");
+      }
+    return boost::make_iterator_range(boost::begin(buffer), pos);
+  }
+};
+
 template <class Tag, unsigned version_major, unsigned version_minor>
 struct async_connection_base;
 
@@ -73,8 +145,10 @@ struct http_async_connection
 
   http_async_connection(resolver_type& resolver, resolve_function resolve,
                         bool follow_redirect, int timeout,
+                        bool remove_chunk_markers,
                         connection_delegate_ptr delegate)
       : timeout_(timeout),
+        remove_chunk_markers_(remove_chunk_markers),
         timer_(resolver.get_io_service()),
         is_timedout_(false),
         follow_redirect_(follow_redirect),
@@ -397,13 +471,23 @@ struct http_async_connection
               callback(make_iterator_range(begin, end), ec);
             } else {
               string_type body_string;
-              std::swap(body_string, this->partial_parsed);
-			  auto it = this->part.begin();
-			  std::advance(it, bytes_transferred);
-              body_string.append(this->part.begin(), it);
-              if (this->is_chunk_encoding) {
-                this->body_promise.set_value(parse_chunk_encoding(body_string));
+              if (this->is_chunk_encoding && remove_chunk_markers_) {
+                for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
+                  auto range = parse_chunk_encoding(
+                    boost::make_iterator_range(this->partial_parsed.data() + i, 
+                                               this->partial_parsed.data() + std::min(i+1024, this->partial_parsed.size())));
+                  body_string.append(boost::begin(range), boost::end(range));
+                }
+                this->partial_parsed.clear();
+                auto range = parse_chunk_encoding(boost::make_iterator_range(this->part.begin(), 
+                                                                             this->part.begin() + bytes_transferred));
+                body_string.append(boost::begin(range), boost::end(range));
+                this->body_promise.set_value(body_string);
               } else {
+                std::swap(body_string, this->partial_parsed);
+                auto it = this->part.begin();
+                std::advance(it, bytes_transferred);
+                body_string.append(this->part.begin(), it);
                 this->body_promise.set_value(body_string);
               }
             }
@@ -520,6 +604,7 @@ struct http_async_connection
   }
 
   int timeout_;
+  bool remove_chunk_markers_;
   boost::asio::steady_timer timer_;
   bool is_timedout_;
   bool follow_redirect_;
@@ -529,6 +614,7 @@ struct http_async_connection
   connection_delegate_ptr delegate_;
   boost::asio::streambuf command_streambuf;
   string_type method;
+  chunk_encoding_parser<Tag> parse_chunk_encoding;
 };
 
 }  // namespace impl

diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp
@@ -312,7 +312,8 @@ class basic_client_facade {
         options.openssl_verify_path(), options.openssl_certificate_file(),
         options.openssl_private_key_file(), options.openssl_ciphers(),
         options.openssl_sni_hostname(), options.openssl_options(),
-        options.io_service(), options.timeout()));
+        options.io_service(), options.timeout(),
+        options.remove_chunk_markers()));
   }
 };
 

diff --git a/boost/network/protocol/http/client/options.hpp b/boost/network/protocol/http/client/options.hpp
@@ -34,7 +34,8 @@ class client_options {
         openssl_options_(0),
         io_service_(),
         always_verify_peer_(true),
-        timeout_(0) {}
+        timeout_(0),
+        remove_chunk_markers_(false) {}
 
   client_options(client_options const& other)
       : cache_resolved_(other.cache_resolved_),
@@ -48,7 +49,8 @@ class client_options {
         openssl_options_(other.openssl_options_),
         io_service_(other.io_service_),
         always_verify_peer_(other.always_verify_peer_),
-        timeout_(other.timeout_) {}
+        timeout_(other.timeout_),
+        remove_chunk_markers_(other.remove_chunk_markers) {}
 
   client_options& operator=(client_options other) {
     other.swap(*this);
@@ -69,6 +71,7 @@ class client_options {
     swap(io_service_, other.io_service_);
     swap(always_verify_peer_, other.always_verify_peer_);
     swap(timeout_, other.timeout_);
+    swap(remove_chunk_markers_, other.remove_chunk_markers_);
   }
 
   /// Specify whether the client should cache resolved endpoints.
@@ -154,6 +157,12 @@ class client_options {
     return *this;
   }
 
+  /// Set an overall timeout for HTTP requests.
+  client_options& remove_chunk_markers(bool v) {
+    remove_chunk_markers_ = v;
+    return *this;
+  }
+
   bool cache_resolved() const { return cache_resolved_; }
 
   bool follow_redirects() const { return follow_redirects_; }
@@ -190,6 +199,8 @@ class client_options {
 
   int timeout() const { return timeout_; }
 
+  bool remove_chunk_markers() const { return remove_chunk_markers_; }
+
  private:
   bool cache_resolved_;
   bool follow_redirects_;
@@ -203,6 +214,7 @@ class client_options {
   std::shared_ptr<boost::asio::io_service> io_service_;
   bool always_verify_peer_;
   int timeout_;
+  bool remove_chunk_markers_;
 };
 
 template <class Tag>

diff --git a/boost/network/protocol/http/client/pimpl.hpp b/boost/network/protocol/http/client/pimpl.hpp
@@ -74,10 +74,12 @@ struct basic_client_impl
                     optional<string_type> const& private_key_file,
                     optional<string_type> const& ciphers,
                     optional<string_type> const& sni_hostname, long ssl_options,
-                    std::shared_ptr<boost::asio::io_service> service, int timeout)
-      : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
-                  service, certificate_filename, verify_path, certificate_file,
-                  private_key_file, ciphers, sni_hostname, ssl_options) {}
+                    std::shared_ptr<boost::asio::io_service> service, int timeout,
+                    bool remove_chunk_markers)
+      : base_type(cache_resolved, follow_redirect, always_verify_peer, timeout, 
+                  remove_chunk_markers, service, certificate_filename, verify_path,
+                  certificate_file, private_key_file, ciphers, sni_hostname,
+                  ssl_options) {}
 
   ~basic_client_impl() = default;
 };

diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp
@@ -40,7 +40,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
   struct connection_impl {
     connection_impl(
         bool follow_redirect, bool always_verify_peer, resolve_function resolve,
-        resolver_type& resolver, bool https, int timeout,
+        resolver_type& resolver, bool https, int timeout, bool remove_chunk_markers,
         optional<string_type> /*unused*/ const& certificate_filename,
         optional<string_type> const& verify_path,
         optional<string_type> const& certificate_file,
@@ -49,8 +49,8 @@ struct async_connection_policy : resolver_policy<Tag>::type {
         optional<string_type> const& sni_hostname, long ssl_options) {
       pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
           new_connection(resolve, resolver, follow_redirect, always_verify_peer,
-                         https, timeout, certificate_filename, verify_path,
-                         certificate_file, private_key_file, ciphers,
+                         https, timeout, remove_chunk_markers, certificate_filename,
+                         verify_path, certificate_file, private_key_file, ciphers,
                          sni_hostname, ssl_options);
     }
 
@@ -87,21 +87,23 @@ struct async_connection_policy : resolver_policy<Tag>::type {
                std::uint16_t port, resolve_completion_function once_resolved) {
           this->resolve(resolver, host, port, once_resolved);
         },
-        resolver, boost::iequals(protocol_, string_type("https")), timeout_,
+        resolver, boost::iequals(protocol_, string_type("https")), timeout_, remove_chunk_markers_,
         certificate_filename, verify_path, certificate_file, private_key_file,
         ciphers, sni_hostname, ssl_options);
   }
 
   void cleanup() {}
 
   async_connection_policy(bool cache_resolved, bool follow_redirect,
-                          int timeout)
+                          int timeout, bool remove_chunk_markers)
       : resolver_base(cache_resolved),
         follow_redirect_(follow_redirect),
-        timeout_(timeout) {}
+        timeout_(timeout),
+        remove_chunk_markers_(remove_chunk_markers) {}
 
   bool follow_redirect_;
   int timeout_;
+  bool remove_chunk_markers_;
 };
 
 }  // namespace http