class Curl::Multi

Public Class Methods

Curl::Multi.default_timeout = 4 → 4 click to toggle source

Get the global default time out for all Curl::Multi Handles.

VALUE ruby_curl_multi_get_default_timeout(VALUE klass) {
  return INT2FIX(cCurlMutiDefaulttimeout);
}
Curl::Multi.default_timeout = 4 → 4 click to toggle source

Set the global default time out for all Curl::Multi Handles. This value is used when libcurl cannot determine a timeout value when calling curl_multi_timeout.

VALUE ruby_curl_multi_set_default_timeout(VALUE klass, VALUE timeout) {
  cCurlMutiDefaulttimeout = FIX2LONG(timeout);
  return timeout;
}
Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|} click to toggle source

will create 2 new files file1.txt and file2.txt

2 files will be opened, and remain open until the call completes

when using the :post or :put method, urls should be a hash, including the individual post fields per post

# File lib/curl/multi.rb, line 183
def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
  errors = []
  procs = []
  files = []
  urls_with_config = []
  url_to_download_paths = {}

  urls.each_with_index do|urlcfg,i|
    if urlcfg.is_a?(Hash)
      url = url[:url]
    else
      url = urlcfg
    end

    if download_paths and download_paths[i]
      download_path = download_paths[i]
    else
      download_path = File.basename(url)
    end

    file = lambda do|dp|
      file = File.open(dp,"wb")
      procs << (lambda {|data| file.write data; data.size })
      files << file
      file
    end.call(download_path)

    if urlcfg.is_a?(Hash)
      urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
    else
      urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
    end
    url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
  end

  if blk
    # when injecting the block, ensure file is closed before yielding
    Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
      info = url_to_download_paths[c.url]
      begin
        file = info[:file]
        files.reject!{|f| f == file }
        file.close
      rescue => e
        errors << e
      end
      blk.call(c,info[:path])
    end
  else
    Curl::Multi.http(urls_with_config, multi_options)
  end

ensure
  files.each {|f|
    begin
      f.close
    rescue => e
      errors << e
    end
  }
  raise errors unless errors.empty?
end
Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location → true) do|easy| click to toggle source
easy
end

Blocking call to fetch multiple url's in parallel.

# File lib/curl/multi.rb, line 11
def get(urls, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls.each do|url|
    url_confs << {:url => url, :method => :get}.merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
end
Curl::Multi.http( [ click to toggle source
{ :url => 'url1', :method → :post,
:post_fields => {'field1' => 'value1', 'field2' → 'value2'} },
{ :url => 'url2', :method → :get,
:follow_location => true, :max_redirects → 3 },
{ :url => 'url3', :method => :put, :put_data → File.open('file.txt','rb') },
{ :url => 'url4', :method → :head }
], {:pipeline → true})

Blocking call to issue multiple HTTP requests with varying verb's.

urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used multi_options: options for the multi handle blk: a callback, that yeilds when a handle is completed

# File lib/curl/multi.rb, line 85
def http(urls_with_config, multi_options={}, &blk)
  m = Curl::Multi.new

  # maintain a sane number of easy handles
  multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10

  free_handles = [] # keep a list of free easy handles

  # configure the multi handle
  multi_options.each { |k,v| m.send("#{k}=", v) }
  callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]

  add_free_handle = proc do|conf, easy|
    c       = conf.dup # avoid being destructive to input
    url     = c.delete(:url)
    method  = c.delete(:method)
    headers = c.delete(:headers)

    easy    = Curl::Easy.new if easy.nil?

    easy.url = url

    # assign callbacks
    callbacks.each do |cb|
      cbproc = c.delete(cb)
      easy.send(cb,&cbproc) if cbproc
    end

    case method
    when :post
      fields = c.delete(:post_fields)
      # set the post post using the url fields
      easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
    when :put
      easy.put_data = c.delete(:put_data)
    when :head
      easy.head = true
    when :delete
      easy.delete = true
    when :get
    else
      # XXX: nil is treated like a GET
    end

    # headers is a special key
    headers.each {|k,v| easy.headers[k] = v } if headers
 
    #
    # use the remaining options as specific configuration to the easy handle
    # bad options should raise an undefined method error
    #
    c.each { |k,v| easy.send("#{k}=",v) }

    easy.on_complete {|curl,code|
      free_handles << curl
      blk.call(curl,code,method) if blk
    }
    m.add(easy)
  end

  max_connects.times do
    conf = urls_with_config.pop
    add_free_handle.call conf, nil
    break if urls_with_config.empty?
  end

  consume_free_handles = proc do
    # as we idle consume free handles
    if urls_with_config.size > 0 && free_handles.size > 0
      easy = free_handles.pop
      conf = urls_with_config.pop
      add_free_handle.call conf, easy
    end
  end

  if urls_with_config.empty?
    m.perform
  else
    until urls_with_config.empty?
      m.perform do
        consume_free_handles.call
      end
      consume_free_handles.call
    end
    free_handles = nil
  end
end
Curl::Multi.new → #&lt;Curl::Easy...&gt; click to toggle source

Create a new Curl::Multi instance

VALUE ruby_curl_multi_new(VALUE klass) {
  VALUE new_curlm;

  ruby_curl_multi *rbcm = ALLOC(ruby_curl_multi);

  rbcm->handle = curl_multi_init();
  if (!rbcm->handle) {
    rb_raise(mCurlErrFailedInit, "Failed to initialize multi handle");
  }

  rbcm->requests = rb_hash_new();

  rbcm->active = 0;
  rbcm->running = 0;

  new_curlm = Data_Wrap_Struct(klass, curl_multi_mark, curl_multi_free, rbcm);

  return new_curlm;
}
Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' → 'value2'}}, click to toggle source
{:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' → 'value2'}},
{:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' → 'value2'}}],
{ :follow_location => true, :multipart_form_post → true },
{:pipeline → true }) do|easy|
easy_handle_on_request_complete
end

Blocking call to POST multiple form's in parallel.

urls_with_config: is a hash of url's pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle

# File lib/curl/multi.rb, line 35
def post(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :post).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end
Curl::Multi.put([{:url => 'url1', :put_data → "some message"}, click to toggle source
{:url => 'url2', :put_data → IO.read('filepath')},
{:url => 'url3', :put_data → "maybe another string or socket?"],
{:follow_location → true},
{:pipeline → true }) do|easy|
easy_handle_on_request_complete
end

Blocking call to POST multiple form's in parallel.

urls_with_config: is a hash of url's pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle

# File lib/curl/multi.rb, line 59
def put(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :put).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

Public Instance Methods

multi = Curl::Multi.new click to toggle source
easy = Curl::Easy.new('url')
add(easy)

Add an easy handle to the multi stack

VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
  CURLMcode mcode;
  ruby_curl_easy *rbce;
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  Data_Get_Struct(easy, ruby_curl_easy, rbce);

  /* setup the easy handle */
  ruby_curl_easy_setup( rbce );

  mcode = curl_multi_add_handle(rbcm->handle, rbce->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    raise_curl_multi_error_exception(mcode);
  }

  rbcm->active++;

  /* Increase the running count, so that the perform loop keeps running.
   * If this number is not correct, the next call to curl_multi_perform will correct it. */
  rbcm->running++;

  rb_hash_aset( rbcm->requests, easy, easy );

  return self;
}
cancel! click to toggle source

Cancels all requests currently being made on this Curl::Multi handle.

static VALUE ruby_curl_multi_cancel(VALUE self) {
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  rb_hash_foreach( rbcm->requests, ruby_curl_multi_cancel_callback, (VALUE)rbcm );
  
  /* for chaining */
  return self;
}
idle? → true or false click to toggle source

Returns whether or not this Curl::Multi handle is processing any requests. E.g. this returns true when multi.requests.length == 0.

static VALUE ruby_curl_multi_idle(VALUE self) {
  ruby_curl_multi *rbcm;
  
  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  if ( FIX2INT( rb_funcall(rbcm->requests, rb_intern("length"), 0) ) == 0 ) {
    return Qtrue;
  } else {
    return Qfalse;
  }
}
multi = Curl::Multi.new click to toggle source
max_connects = 800

Set the max connections in the cache for a multi handle

static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAXCONNECTS
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2INT(count));
#endif

  return count;
}
multi = Curl::Multi.new click to toggle source
easy1 = Curl::Easy.new('url')
easy2 = Curl::Easy.new('url')
add(easy1)
add(easy2)
perform do
# while idle other code my execute here
end

Run multi handles, looping selecting when data can be transfered

VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) {
  CURLMcode mcode;
  ruby_curl_multi *rbcm;
  int maxfd, rc;
  fd_set fdread, fdwrite, fdexcep;
#ifdef _WIN32
  fd_set crt_fdread, crt_fdwrite, crt_fdexcep;
#endif
  long timeout_milliseconds;
  struct timeval tv = {0, 0};
  VALUE block = Qnil;
#if defined(HAVE_RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
  struct _select_set fdset_args;
#endif

  rb_scan_args(argc, argv, "0&", &block);

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  timeout_milliseconds = cCurlMutiDefaulttimeout;

  rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
  rb_curl_multi_read_info( self, rbcm->handle );
  if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
 
  do {
    while (rbcm->running) {

#ifdef HAVE_CURL_MULTI_TIMEOUT
      /* get the curl suggested time out */
      mcode = curl_multi_timeout(rbcm->handle, &timeout_milliseconds);
      if (mcode != CURLM_OK) {
        raise_curl_multi_error_exception(mcode);
      }
#else
      /* libcurl doesn't have a timeout method defined, initialize to -1 we'll pick up the default later */
      timeout_milliseconds = -1;
#endif

      if (timeout_milliseconds == 0) { /* no delay */
        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
        rb_curl_multi_read_info( self, rbcm->handle );
        if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
        continue;
      }

      if (timeout_milliseconds < 0 || timeout_milliseconds > cCurlMutiDefaulttimeout) {
        timeout_milliseconds = cCurlMutiDefaulttimeout; /* libcurl doesn't know how long to wait, use a default timeout */
                                                        /* or buggy versions libcurl sometimes reports huge timeouts... let's cap it */
      }

      tv.tv_sec  = 0; /* never wait longer than 1 second */
      tv.tv_usec = (int)(timeout_milliseconds * 1000); /* XXX: int is the right type for OSX, what about linux? */

      FD_ZERO(&fdread);
      FD_ZERO(&fdwrite);
      FD_ZERO(&fdexcep);

      /* load the fd sets from the multi handle */
      mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
      if (mcode != CURLM_OK) {
        raise_curl_multi_error_exception(mcode);
      }

#ifdef _WIN32
      create_crt_fd(&fdread, &crt_fdread);
      create_crt_fd(&fdwrite, &crt_fdwrite);
      create_crt_fd(&fdexcep, &crt_fdexcep);
#endif

#if defined(HAVE_RB_THREAD_BLOCKING_REGION) || defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL)
      fdset_args.maxfd = maxfd+1;
      fdset_args.fdread = &fdread;
      fdset_args.fdwrite = &fdwrite;
      fdset_args.fdexcep = &fdexcep;
      fdset_args.tv = &tv;
#ifdef HAVE_RB_THREAD_CALL_WITHOUT_GVL
      rc = (int)(VALUE) rb_thread_call_without_gvl((void *(*)(void *))curb_select, &fdset_args, RUBY_UBF_IO, 0);
#elif HAVE_RB_THREAD_BLOCKING_REGION
      rc = rb_thread_blocking_region(curb_select, &fdset_args, RUBY_UBF_IO, 0);
#elif HAVE_RB_THREAD_FD_SELECT
      rc = rb_thread_fd_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
#else
      rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
#endif

#endif

#ifdef _WIN32
      cleanup_crt_fd(&fdread, &crt_fdread);
      cleanup_crt_fd(&fdwrite, &crt_fdwrite);
      cleanup_crt_fd(&fdexcep, &crt_fdexcep);
#endif

      switch(rc) {
      case -1:
        if(errno != EINTR) {
          rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
          break;
        }
      case 0: /* timeout */
      default: /* action */
        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
        rb_curl_multi_read_info( self, rbcm->handle );
        if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
        break;
      }
    }

  } while( rbcm->running );

  rb_curl_multi_read_info( self, rbcm->handle );
  if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
    
  return Qtrue;
}
multi = Curl::Multi.new click to toggle source
pipeline = true

Pass a long set to 1 to enable or 0 to disable. Enabling pipelining on a multi handle will make it attempt to perform HTTP Pipelining as far as possible for transfers using this handle. This means that if you add a second request that can use an already existing connection, the second request will be “piped” on the same connection rather than being executed in parallel. (Added in 7.16.0)

static VALUE ruby_curl_multi_pipeline(VALUE self, VALUE onoff) {
#ifdef HAVE_CURLMOPT_PIPELINING
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_PIPELINING, onoff == Qtrue ? 1 : 0);
#endif
  return onoff;
}
multi = Curl::Multi.new click to toggle source
easy = Curl::Easy.new('url')
add(easy)

# sometime later multi.remove(easy)

Remove an easy handle from a multi stack.

Will raise an exception if the easy handle is not found

VALUE ruby_curl_multi_remove(VALUE self, VALUE easy) {
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  rb_curl_multi_remove(rbcm,easy);

  return self;
}
requests → [#&lt;Curl::Easy...&gt;, ...] click to toggle source

Returns an array containing all the active requests on this Curl::Multi object.

static VALUE ruby_curl_multi_requests(VALUE self) {
  ruby_curl_multi *rbcm;
  VALUE result_array;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  result_array = rb_ary_new();
  
  /* iterate over the requests hash, and stuff references into the array. */
  rb_hash_foreach(rbcm->requests, ruby_curl_multi_requests_callback, result_array);
  
  return result_array;
}