Ticket #30: mongrel_changes.diff

lib/mongrel/configurator.rb

@@ -136 +136 @@
-      opts[:throttle] ||= 0
-      opts[:timeout] ||= 60
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-      @listener_name = "#{opts[:host]}:#{opts[:port]}"
-      @listeners[@listener_name] = @listener
-
@@ -372 +383 @@
-
-      unless RUBY_PLATFORM =~ /djgpp|(cyg|ms|bcc)win|mingw/
-        # graceful shutdown
-
+pid #{$$}: 
-        # debug mode
-
+pid #{$$}: 
-        # restart
-
+pid #{$$}: 
-
-        Mongrel.log(:notice, "Signals ready.  TERM => stop.  USR2 => restart.  INT => stop (no restart).")
-      else
@@ -387 +398 @@
-  end
-
-end
+# vim:shiftwidth=2:tabstop=2:expandtab:ft=ruby

lib/mongrel.rb

@@ +1 @@
+# Copyright (c) 2005 Zed A. Shaw
+# Copyright (c) 2008 Raritan Computer, Inc (Brian Weaver <brian.weaver@raritan.com>)
-# Ruby
-require 'socket'
-require 'tempfile'
@@ -47 +49 @@
-    attr_accessor :http_body
-  end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-    attr_accessor :logger
+    attr_reader   :acceptor
+    attr_accessor :classifier
+    attr_reader   :domain
+    attr_reader   :host
+    attr_reader   :port
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-      @classifier = URIClassifier.new
-
-
-
-
-
-
-
+
-    end
-    
-    # Does the majority of the IO processing.  It has been written in Ruby using
@@ -117 +74 @@
-        request = nil
-        data = client.readpartial(Const::CHUNK_SIZE)
-        nparsed = 0
-
+
-        # Assumption: nparsed will always be less since data will get filled with more
-        # after each parsing.  If it doesn't get more then there was a problem
-        # with the read operation on the client socket.  Effect is to stop processing when the
@@ -185 +142 @@
-            end
-          end
-        end
-
+
+
-        client.close rescue nil
-      rescue HttpParserError => e
-#{Time.now.httpdate}: 
-#{Time.now.httpdate}: 
+
+
-        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4
-        client.write(Const::ERROR_400_RESPONSE)
-
+
+
-        reap_dead_workers('too many files')
-      rescue Object => e
-#{Time.now.httpdate}: 
+
-        Mongrel.log(:error, e.backtrace.join("\n"))
-      ensure
-        begin
-          client.close
-
+
+
-          # Already closed
-        rescue Object => e
-#{Time.now.httpdate}: 
+
-          Mongrel.log(:error, e.backtrace.join("\n"))
-        end
-        request.body.delete if request and request.body.class == Tempfile
-      end
-    end
-
+    def reap_dead_workers(reason=nil)
+      # override in subclass!
+    end
+
+    def configure_socket_options
+      case RUBY_PLATFORM
+      when /linux/
+        # 9 is currently TCP_DEFER_ACCEPT
+        $tcp_defer_accept_opts = [Socket::SOL_TCP, 9, 1]
+        $tcp_cork_opts = [Socket::SOL_TCP, 3, 1]
+      when /freebsd(([1-4]\..{1,2})|5\.[0-4])/
+        # Do nothing, just closing a bug when freebsd <= 5.4
+      when /freebsd/
+        # Use the HTTP accept filter if available.
+        # The struct made by pack() is defined in /usr/include/sys/socket.h as accept_filter_arg
+        unless `/sbin/sysctl -nq net.inet.accf.http`.empty?
+          $tcp_defer_accept_opts = [Socket::SOL_SOCKET, Socket::SO_ACCEPTFILTER, ['httpready', nil].pack('a16a240')]
+        end
+      end
+    end
+
+    # Runs the thing.  It returns the thread used so you can "join" it.  You can also
+    # access the HttpServer::acceptor attribute to get the thread later.
+    def run
+      raise RuntimeError, "This method must be overridden by a subclass"
+    end
+
+    # Simply registers a handler with the internal URIClassifier.  When the URI is
+    # found in the prefix of a request then your handler's HttpHandler::process method
+    # is called.  See Mongrel::URIClassifier#register for more information.
+    #
+    # If you set in_front=true then the passed in handler will be put in the front of the list
+    # for that particular URI. Otherwise it's placed at the end of the list.
+    def register(uri, handler, in_front=false)
+      begin
+        @classifier.register(uri, [handler])
+      rescue URIClassifier::RegistrationError
+        handlers = @classifier.resolve(uri)[2]
+        method_name = in_front ? 'unshift' : 'push'
+        handlers.send(method_name, handler)
+      end
+      handler.listener = self
+    end
+
+    # Removes any handlers registered at the given URI.  See Mongrel::URIClassifier#unregister
+    # for more information.  Remember this removes them *all* so the entire
+    # processing chain goes away.
+    def unregister(uri)
+      @classifier.unregister(uri)
+    end
+
+    def stop(synchronous=false)
+      #Mongrel.log(:debug, "stop invoked for #{self.inspect}")
+      # this appears to get invoked once for each signal, but after
+      # the first signal the 'acceptor' appear to be nil. WTF?
+      return if @acceptor.nil?
+      @acceptor.raise(StopServer.new)
+
+      if synchronous
+        sleep(0.5) while @acceptor.alive?
+      end
+    end
+
+  end
+
+  # This is the main driver of Mongrel, while the Mongrel::HttpParser and Mongrel::URIClassifier
+  # make up the majority of how the server functions.  It's a very simple class that just
+  # has a thread accepting connections and a simple HttpServer.process_client function
+  # to do the heavy lifting with the IO and Ruby.  
+  #
+  # You use it by doing the following:
+  #
+  #   server = 2("0.0.0.0", 3000)
+  #   server.register("/stuff", MyNiftyHandler.new)
+  #   server.run.join
+  #
+  # The last line can be just server.run if you don't want to join the thread used.
+  # If you don't though Ruby will mysteriously just exit on you.
+  #
+  # Ruby's thread implementation is "interesting" to say the least.  Experiments with
+  # *many* different types of IO processing simply cannot make a dent in it.  Future
+  # releases of Mongrel will find other creative ways to make threads faster, but don't
+  # hold your breath until Ruby 1.9 is actually finally useful.
+  class HttpServer < Server
+    attr_reader :workers
+    attr_reader :throttle
+    attr_reader :timeout
+    attr_reader :num_processors
+
+    # Creates a working server on host:port (strange things happen if port isn't a Number).
+    # Use HttpServer::run to start the server and HttpServer.acceptor.join to 
+    # join the thread that's processing incoming requests on the socket.
+    #
+    # The num_processors optional argument is the maximum number of concurrent
+    # processors to accept, anything over this is closed immediately to maintain
+    # server processing performance.  This may seem mean but it is the most efficient
+    # way to deal with overload.  Other schemes involve still parsing the client's request
+    # which defeats the point of an overload handling system.
+    # 
+    # The throttle parameter is a sleep timeout (in hundredths of a second) that is placed between 
+    # socket.accept calls in order to give the server a cheap throttle time.  It defaults to 0 and
+    # actually if it is 0 then the sleep is not done at all.
+    def initialize(host, port, num_processors=950, throttle=0, timeout=60, log=nil, log_level=:debug)
+      
+      @host   = host
+      @port   = port
+      @domain = "tcp"
+
+      super(log, log_level)
+
+      @socket   = TCPServer.new(host, port) 
+      @workers  = ThreadGroup.new
+      @throttle = throttle
+      @timeout  = timeout
+      @num_processors = num_processors
+
+    end
+    
-    # Used internally to kill off any worker threads that have taken too long
-    # to complete processing.  Only called if there are too many processors
-    # currently servicing.  It returns the count of workers still active
-    # after the reap is done.  It only runs if there are workers to reap.
-    def reap_dead_workers(reason='unknown')
+
-      if @workers.list.length > 0
-
-
+
+
+
-        mark = Time.now
-        @workers.list.each do |worker|
-          worker[:started_on] = Time.now if not worker[:started_on]
-
-          if mark - worker[:started_on] > @timeout + @throttle
-#{Time.now.httpdate}: 
+
-            worker.raise(TimeoutError.new(error_msg))
-          end
-        end
+
-      end
-
-return 
+
-    end
-
-    # Performs a wait on all the currently running threads and kills any that take
@@ -238 +319 @@
-    # that much longer.
-    def graceful_shutdown
-      while reap_dead_workers("shutdown") > 0
-#{Time.now.httpdate}: 
+
-        sleep @timeout / 10
-      end
-    end
-
-    def configure_socket_options
-      case RUBY_PLATFORM
-      when /linux/
-        # 9 is currently TCP_DEFER_ACCEPT
-        $tcp_defer_accept_opts = [Socket::SOL_TCP, 9, 1]
-        $tcp_cork_opts = [Socket::SOL_TCP, 3, 1]
-      when /freebsd(([1-4]\..{1,2})|5\.[0-4])/
-        # Do nothing, just closing a bug when freebsd <= 5.4
-      when /freebsd/
-        # Use the HTTP accept filter if available.
-        # The struct made by pack() is defined in /usr/include/sys/socket.h as accept_filter_arg
-        unless `/sbin/sysctl -nq net.inet.accf.http`.empty?
-          $tcp_defer_accept_opts = [Socket::SOL_SOCKET, Socket::SO_ACCEPTFILTER, ['httpready', nil].pack('a16a240')]
-        end
-      end
-    end
-    
-    # Runs the thing.  It returns the thread used so you can "join" it.  You can also
-    # access the HttpServer::acceptor attribute to get the thread later.
-    def run
@@ -279 +343 @@
-
-              num_workers = @workers.list.length
-              if num_workers >= @num_processors
-#{Time.now.httpdate}: 
+
-                client.close rescue nil
-                reap_dead_workers("max processors")
-              else
@@ -301 +365 @@
-              # client closed the socket even before accept
-              client.close rescue nil
-            rescue Object => e
-#{Time.now.httpdate}: 
+
-              Mongrel.log(:error, e.backtrace.join("\n"))
-            end
-          end
-          graceful_shutdown
-        ensure
-          @socket.close
-          # Mongrel.log(:error, "#{Time.now.httpdate}: Closed socket.")
-        end
-      end
-
-return 
+
-    end
+  end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-      end
-
+
+
-    end
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    end
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
-      end
+
+      @acceptor = Thread.new do
+        begin
+          while not @terminate
+            begin
+              @children.each_key do |pid|
+                begin
+                  do_evict = false
+                  do_evict = true if @children[pid].closed? or Process.waitpid(pid, Process::WNOHANG) 
+                rescue Errno::ECHILD
+                  do_evice = true
+                rescue Object => e
+                  Mongrel.log(:error, "Unhandled exception #{e.inspect}.")
+                  Mongrel.log(:error, e.backtrace.join("\n"))
+                end
+
+                evict_child(pid) if do_evict
+              end
+
+              readfds = [ @tcpsocket ] + @children.values 
+              r,w,e  = Kernel.select(readfds, nil, nil, 60)
+
+              # check to see if anyone wants to talk to us. If no one is talking
+              # and we have more than the necessary number of children then signal
+              # one to terminate.
+              #
+              evict_child(@children.keys.first) if (r.nil? or r.empty?) and @children.length > @min_children
+              next if r.nil?
+
+              # OK, someone is talking. Find out who and process the client.
+              # Delay the processing of the TCP server socket until all the children
+              # are finished. This allows a client to become available before processing
+              # the HTTP connection.
+              #
+              flag_http = false
+              r.each do |io|
+                if io == @tcpsocket # a new connection from apache or the outside world
+                  flag_http = true  # defer
+                else                # one of the children
+                  begin
+                    msg = io.readline
+                    msg.chomp!
+                    if msg =~ /^READY\s+(\d+)$/
+                      pid = $1.to_i
+                      if @busy[pid].respond_to?(:close)
+                        @busy[pid].close rescue nil
+                      end
+                      @busy[pid] = false
+                    elsif msg =~ /CLOSED\s+(\d+)/
+                      evict_child($1.to_i)
+                    end
+                  rescue EOFError => ex
+                    Mongrel.log(:error,ex.message)
+                    @children.each_key do |pid|
+                      evict_child(pid) if @children[pid] == io
+                    end
+                  end
+                end
+              end
+
+              next unless flag_http
+
+              client = @tcpsocket.accept
+              @busy.each_pair do |pid,busy|
+                unless busy
+                  io = @children[pid]
+                  io.send_io(client)
+                  io.flush
+
+                  @busy[pid] = client
+                  client = nil
+                  break
+                end
+              end
+
+              if client and not @max_children.nil? and @max_children == @children.length
+                Mongrel.log(:error, "Maximum number of thread exceeded, closing request")
+                client.close rescue nil
+                client = nil
+              end
+
+              next unless client
+
+              Mongrel.log(:debug, "No avaialble child found, spinning up a new child")
+
+              # spin up a new one
+              #
+              pid, socket = start_child
+              @children[pid] = socket
+              socket.readline
+              socket.send_io(client)
+              socket.flush
+              @busy[pid] = client
+
+            rescue StopServer
+              @terminate = true
+            rescue Object => e
+              Mongrel.log(:error, "Unhandled listen loop exception #{e.inspect}.")
+              Mongrel.log(:error, e.backtrace.join("\n"))
+            end
+          end
+        ensure
+          @tcpsocket.close  rescue nil
+          @children.each_key { |pid| evict_child(pid) }
+        end
+      end # end thread
+
+      @acceptor
-    end
-
-  end
-end
-
@@ -357 +624 @@
-
-$LOAD_PATH.unshift 'projects/mongrel_experimental/lib/'
-Mongrel::Gems.require 'mongrel_experimental', ">=#{Mongrel::Const::MONGREL_VERSION}"
+
+# vim:expandtab:shiftwidth=2:tabstop=2

bin/mongrel_rails

@@ -1 +1 @@
-# Copyright (c) 2005 Zed A. Shaw
+# Copyright (c) 2008 Raritan Computer, Inc (Brian Weaver <brian.weaver@raritan.com>)
-# You can redistribute it and/or modify it under the same terms as Ruby.
-#
-# Additional work donated by contributors.  See http://mongrel.rubyforge.org/attributions.html
@@ -27 +28 @@
-        ['-p', '--port PORT', "Which port to bind to", :@port, 3000],
-        ['-a', '--address ADDR', "Address to bind to", :@address, "0.0.0.0"],
-        ['-l', '--log FILE', "Where to write log messages", :@log_file, "log/mongrel.log"],
+        ['-L', '--log-level (debug|notice|warn|info|error)', "The logging level", :@log_level, 'debug'],
-        ['-P', '--pid FILE', "Where to write the PID", :@pid_file, "log/mongrel.pid"],
-        ['-n', '--num-procs INT', "Number of processors active before clients denied", :@num_processors, 1024],
-        ['-o', '--timeout TIME', "Time to wait (in seconds) before killing a stalled thread", :@timeout, 60],
@@ -38 +40 @@
-        ['-C', '--config PATH', "Use a config file", :@config_file, nil],
-        ['-S', '--script PATH', "Load the given file as an extra config script", :@config_script, nil],
-        ['-G', '--generate PATH', "Generate a config file for use with -C", :@generate, nil],
+        ['-U', '--unixmaster', "Run the rails as a master-slave server", :@unixmaster, false],
+        ['-N', '--children COUNT', "The default number of unix slaves to start", :@min_children, 5],
+        ['-M', '--max-children COUNT', "The maximum number of unix slaves to start", :@max_children, nil],
-        ['', '--user USER', "User to run as", :@user, nil],
-        ['', '--group GROUP', "Group to run as", :@group, nil],
-        ['', '--prefix PATH', "URL prefix for Rails app", :@prefix, nil]
@@ -69 +74 @@
-      valid_user? @user if @user
-      valid_group? @group if @group
-
+      @log_level = @log_level.to_sym unless @log_level.nil? or @log_level.is_a?(Symbol)
+
-      return @valid
-    end
-
@@ -95 +102 @@
-          Mongrel.log("Settings loaded from #{@config_file} (they override command line).") if @config_file
-        end
-
-log/mongrel-#{defaults[:host]}-#{defaults[:port]}.log
+#{defaults[:log_file]}
-
-        listener do
-          mime = {}
@@ -180 +187 @@
-
-    def config_keys
-      @config_keys ||=
-pid_file environment docroot mime_map daemon debug includes config_script num_processors num_procs timeout throttle user group prefix
+log_level pid_file environment docroot mime_map daemon debug includes config_script num_processors num_procs timeout throttle user group prefix unixmaster min_children max_children
-    end
-
-    def settings
@@ -282 +289 @@
-if not Mongrel::Command::Registry.instance.run ARGV
-  exit 1
-end
+
+# vim:shiftwidth=2:tabstop=2:expandtab:ft=ruby