class Curl::Multi
Public Class Methods
Get the global default autoclose setting for all Curl::Multi
Handles.
VALUE ruby_curl_multi_get_autoclose(VALUE klass) { return cCurlMutiAutoClose == 1 ? Qtrue : Qfalse; }
Automatically close open connections after each request. Otherwise, the connection will remain open for reuse until the next GC
VALUE ruby_curl_multi_set_autoclose(VALUE klass, VALUE onoff) { cCurlMutiAutoClose = ((onoff == Qtrue) ? 1 : 0); return onoff; }
Get the global default time out for all Curl::Multi
Handles.
VALUE ruby_curl_multi_get_default_timeout(VALUE klass) { return LONG2NUM(cCurlMutiDefaulttimeout); }
Set the global default time out for all Curl::Multi
Handles. This value is used when libcurl cannot determine a timeout value when calling curl_multi_timeout.
VALUE ruby_curl_multi_set_default_timeout(VALUE klass, VALUE timeout) { cCurlMutiDefaulttimeout = NUM2LONG(timeout); return timeout; }
Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']
){|c|}
will create 2 new files file1.txt and file2.txt
2 files will be opened, and remain open until the call completes
when using the :post or :put method, urls should be a hash, including the individual post fields per post
# File lib/curl/multi.rb, line 187 def download(urls,easy_options={},multi_options={},download_paths=nil,&blk) errors = [] procs = [] files = [] urls_with_config = [] url_to_download_paths = {} urls.each_with_index do|urlcfg,i| if urlcfg.is_a?(Hash) url = url[:url] else url = urlcfg end if download_paths and download_paths[i] download_path = download_paths[i] else download_path = File.basename(url) end file = lambda do|dp| file = File.open(dp,"wb") procs << (lambda {|data| file.write data; data.size }) files << file file end.call(download_path) if urlcfg.is_a?(Hash) urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options)) else urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options) end url_to_download_paths[url] = {:path => download_path, :file => file} # store for later end if blk # when injecting the block, ensure file is closed before yielding Curl::Multi.http(urls_with_config, multi_options) do |c,code,method| info = url_to_download_paths[c.url] begin file = info[:file] files.reject!{|f| f == file } file.close rescue => e errors << e end blk.call(c,info[:path]) end else Curl::Multi.http(urls_with_config, multi_options) end ensure files.each {|f| begin f.close rescue => e errors << e end } if errors.any? de = Curl::Multi::DownloadError.new de.errors = errors raise de end end
Blocking call to fetch multiple url’s in parallel.
# File lib/curl/multi.rb, line 14 def get(urls, easy_options={}, multi_options={}, &blk) url_confs = [] urls.each do|url| url_confs << {:url => url, :method => :get}.merge(easy_options) end self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk } end
{ :url => 'url1', :method => :post, :post_fields => {'field1' => 'value1', 'field2' => 'value2'} }, { :url => 'url2', :method => :get, :follow_location => true, :max_redirects => 3 }, { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') }, { :url => 'url4', :method => :head }
], {:pipeline => Curl::CURLPIPE_HTTP1})
Blocking call to issue multiple HTTP requests with varying verb’s.
urls_with_config: is a hash of url’s pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used multi_options: options for the multi handle blk: a callback, that yeilds when a handle is completed
# File lib/curl/multi.rb, line 88 def http(urls_with_config, multi_options={}, &blk) m = Curl::Multi.new # maintain a sane number of easy handles multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10 free_handles = [] # keep a list of free easy handles # configure the multi handle multi_options.each { |k,v| m.send("#{k}=", v) } callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_redirect,:on_body,:on_header] add_free_handle = proc do|conf, easy| c = conf.dup # avoid being destructive to input url = c.delete(:url) method = c.delete(:method) headers = c.delete(:headers) easy = Curl::Easy.new if easy.nil? easy.url = url # assign callbacks callbacks.each do |cb| cbproc = c.delete(cb) easy.send(cb,&cbproc) if cbproc end case method when :post fields = c.delete(:post_fields) # set the post post using the url fields easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&') when :put easy.put_data = c.delete(:put_data) when :head easy.head = true when :delete easy.delete = true when :get else # XXX: nil is treated like a GET end # headers is a special key headers.each {|k,v| easy.headers[k] = v } if headers # # use the remaining options as specific configuration to the easy handle # bad options should raise an undefined method error # c.each { |k,v| easy.send("#{k}=",v) } easy.on_complete {|curl| free_handles << curl blk.call(curl,curl.response_code,method) if blk } m.add(easy) end max_connects.times do conf = urls_with_config.pop add_free_handle.call(conf, nil) if conf break if urls_with_config.empty? end consume_free_handles = proc do # as we idle consume free handles if urls_with_config.size > 0 && free_handles.size > 0 easy = free_handles.pop conf = urls_with_config.pop add_free_handle.call(conf, easy) if conf end end if urls_with_config.empty? m.perform else until urls_with_config.empty? m.perform do consume_free_handles.call end consume_free_handles.call end free_handles = nil end end
Create a new Curl::Multi
instance
VALUE ruby_curl_multi_new(VALUE klass) { ruby_curl_multi *rbcm = ALLOC(ruby_curl_multi); ruby_curl_multi_init(rbcm); /* * The mark routine will be called by the garbage collector during its ``mark'' phase. * If your structure references other Ruby objects, then your mark function needs to * identify these objects using rb_gc_mark(value). If the structure doesn't reference * other Ruby objects, you can simply pass 0 as a function pointer. */ return Data_Wrap_Struct(klass, 0, curl_multi_free, rbcm); }
Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}, {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}, {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}], { :follow_location => true, :multipart_form_post => true }, {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy| easy_handle_on_request_complete end
Blocking call to POST multiple form’s in parallel.
urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi
handle
# File lib/curl/multi.rb, line 38 def post(urls_with_config, easy_options={}, multi_options={}, &blk) url_confs = [] urls_with_config.each do|uconf| url_confs << uconf.merge(:method => :post).merge(easy_options) end self.http(url_confs, multi_options) {|c,code,method| blk.call(c) } end
Curl::Multi.put([{:url => 'url1', :put_data => "some message"}, {:url => 'url2', :put_data => IO.read('filepath')}, {:url => 'url3', :put_data => "maybe another string or socket?"], {:follow_location => true}, {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy| easy_handle_on_request_complete end
Blocking call to POST multiple form’s in parallel.
urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi
handle
# File lib/curl/multi.rb, line 62 def put(urls_with_config, easy_options={}, multi_options={}, &blk) url_confs = [] urls_with_config.each do|uconf| url_confs << uconf.merge(:method => :put).merge(easy_options) end self.http(url_confs, multi_options) {|c,code,method| blk.call(c) } end
Public Instance Methods
Add an easy handle to the multi stack
VALUE ruby_curl_multi_add(VALUE self, VALUE easy) { CURLMcode mcode; ruby_curl_easy *rbce; ruby_curl_multi *rbcm; Data_Get_Struct(self, ruby_curl_multi, rbcm); Data_Get_Struct(easy, ruby_curl_easy, rbce); /* setup the easy handle */ ruby_curl_easy_setup( rbce ); mcode = curl_multi_add_handle(rbcm->handle, rbce->curl); if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) { raise_curl_multi_error_exception(mcode); } rbcm->active++; /* Increase the running count, so that the perform loop keeps running. * If this number is not correct, the next call to curl_multi_perform will correct it. */ rbcm->running++; /* track a reference to associated multi handle */ rbce->multi = self; return self; }
multi.close after closing the multi handle all connections will be closed and the handle will no longer be usable
VALUE ruby_curl_multi_close(VALUE self) { ruby_curl_multi *rbcm; Data_Get_Struct(self, ruby_curl_multi, rbcm); curl_multi_cleanup(rbcm->handle); ruby_curl_multi_init(rbcm); return self; }
# sometime later multi.remove(easy)
Remove an easy handle from a multi stack.
Will raise an exception if the easy handle is not found
VALUE ruby_curl_multi_remove(VALUE self, VALUE rb_easy_handle) { ruby_curl_multi *rbcm; Data_Get_Struct(self, ruby_curl_multi, rbcm); rb_curl_multi_remove(rbcm, rb_easy_handle); return self; }
# File lib/curl/multi.rb, line 269 def add(easy) return self if requests[easy.object_id] requests[easy.object_id] = easy _add(easy) self end
# File lib/curl/multi.rb, line 255 def cancel! requests.each do |_,easy| remove(easy) end end
# File lib/curl/multi.rb, line 283 def close requests.values.each {|easy| _remove(easy) } @requests = {} _close self end
# File lib/curl/multi.rb, line 261 def idle? requests.empty? end
Set the max connections in the cache for a multi handle
static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) { #ifdef HAVE_CURLMOPT_MAXCONNECTS ruby_curl_multi *rbcm; Data_Get_Struct(self, ruby_curl_multi, rbcm); curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2LONG(count)); #endif return count; }
multi.add(easy1) multi.add(easy2)
multi.perform do
# while idle other code my execute here
end
Run multi handles, looping selecting when data can be transfered
VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) { CURLMcode mcode; ruby_curl_multi *rbcm; int maxfd, rc = -1; fd_set fdread, fdwrite, fdexcep; #ifdef _WIN32 fd_set crt_fdread, crt_fdwrite, crt_fdexcep; #endif long timeout_milliseconds; struct timeval tv = {0, 0}; struct timeval tv_100ms = {0, 100000}; VALUE block = Qnil; #if defined(HAVE_RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL) struct _select_set fdset_args; #endif rb_scan_args(argc, argv, "0&", &block); Data_Get_Struct(self, ruby_curl_multi, rbcm); timeout_milliseconds = cCurlMutiDefaulttimeout; // Run curl_multi_perform for the first time to get the ball rolling rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) ); // Check the easy handles for new messages one more time before yielding // control to passed ruby block. // // This call will block until all queued messages are processed and if any // handle completed the transfer we will run the on_complete callback here too. rb_curl_multi_read_info( self, rbcm->handle ); // There are no more messages to handle by curl and we can run the ruby block // passed to perform method. // When the block completes curl will resume. if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self); } do { while (rbcm->running) { #ifdef HAVE_CURL_MULTI_TIMEOUT /* get the curl suggested time out */ mcode = curl_multi_timeout(rbcm->handle, &timeout_milliseconds); if (mcode != CURLM_OK) { raise_curl_multi_error_exception(mcode); } #else /* libcurl doesn't have a timeout method defined, initialize to -1 we'll pick up the default later */ timeout_milliseconds = -1; #endif if (timeout_milliseconds == 0) { /* no delay */ rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) ); rb_curl_multi_read_info( self, rbcm->handle ); if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self); } continue; } if (timeout_milliseconds < 0 || timeout_milliseconds > cCurlMutiDefaulttimeout) { timeout_milliseconds = cCurlMutiDefaulttimeout; /* libcurl doesn't know how long to wait, use a default timeout */ /* or buggy versions libcurl sometimes reports huge timeouts... let's cap it */ } tv.tv_sec = 0; /* never wait longer than 1 second */ tv.tv_usec = (int)(timeout_milliseconds * 1000); /* XXX: int is the right type for OSX, what about linux? */ FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); /* load the fd sets from the multi handle */ mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd); if (mcode != CURLM_OK) { raise_curl_multi_error_exception(mcode); } if (maxfd == -1) { /* libcurl recommends sleeping for 100ms */ rb_thread_wait_for(tv_100ms); rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) ); rb_curl_multi_read_info( self, rbcm->handle ); if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self); } continue; } #ifdef _WIN32 create_crt_fd(&fdread, &crt_fdread); create_crt_fd(&fdwrite, &crt_fdwrite); create_crt_fd(&fdexcep, &crt_fdexcep); #endif #if (defined(HAVE_RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)) fdset_args.maxfd = maxfd+1; fdset_args.fdread = &fdread; fdset_args.fdwrite = &fdwrite; fdset_args.fdexcep = &fdexcep; fdset_args.tv = &tv; #endif #ifdef HAVE_RB_THREAD_CALL_WITHOUT_GVL rc = (int)(VALUE) rb_thread_call_without_gvl((void *(*)(void *))curb_select, &fdset_args, RUBY_UBF_IO, 0); #elif HAVE_RB_THREAD_BLOCKING_REGION rc = rb_thread_blocking_region(curb_select, &fdset_args, RUBY_UBF_IO, 0); #elif HAVE_RB_THREAD_FD_SELECT rc = rb_thread_fd_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv); #else rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv); #endif #ifdef _WIN32 cleanup_crt_fd(&fdread, &crt_fdread); cleanup_crt_fd(&fdwrite, &crt_fdwrite); cleanup_crt_fd(&fdexcep, &crt_fdexcep); #endif switch(rc) { case -1: if(errno != EINTR) { rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno)); break; } case 0: /* timeout */ default: /* action */ rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) ); rb_curl_multi_read_info( self, rbcm->handle ); if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self); } break; } } } while( rbcm->running ); rb_curl_multi_read_info( self, rbcm->handle ); if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self); } if (cCurlMutiAutoClose == 1) { rb_funcall(self, rb_intern("close"), 0); } return Qtrue; }
Pass a long set to 1 for HTTP/1.1 pipelining, 2 for HTTP/2 multiplexing, or 0 to disable.
Enabling pipelining on a multi handle will make it attempt to perform HTTP Pipelining as
far as possible for transfers using this handle. This means that if you add a second request that can use an already existing connection, the second request will be “piped” on the same connection rather than being executed in parallel. (Added in 7.16.0, multiplex added in 7.43.0)
static VALUE ruby_curl_multi_pipeline(VALUE self, VALUE method) { #ifdef HAVE_CURLMOPT_PIPELINING ruby_curl_multi *rbcm; long value; if (method == Qtrue) { value = 1; } else if (method == Qfalse) { value = 0; } else { value = NUM2LONG(method); } Data_Get_Struct(self, ruby_curl_multi, rbcm); curl_multi_setopt(rbcm->handle, CURLMOPT_PIPELINING, value); #endif return method == Qtrue ? 1 : 0; }
# File lib/curl/multi.rb, line 276 def remove(easy) return self if !requests[easy.object_id] requests.delete(easy.object_id) _remove(easy) self end
# File lib/curl/multi.rb, line 265 def requests @requests ||= {} end