c# - Efficient use of CPU idle time for parallel URL processing -

we have job service in production server triggering during interval of 2 minutes of timer.
service collecting 20 tasks database, creates 20 urls(same host different parameters); fire them in parallel using thread pool threads, , wait response.
urls loop back. ie, destination website hosted in same server.

process flow follows:

1. if task simple, url response job service within seconds.     job service has wait 2 minutes pick next 20 jobs.     cpu idle here.     how can efficiently utilize cpu idle time processing more jobs…?  2. if task long running, job service wait max 2 minutes     response, if not receiving response; service     pick next 20 jobs process. result jobs queued     , cpu usage go high.     how prevent such situation….? 

instead of timer , picking jobs in regular intervals, have other efficient methods process jobs..?
monitor iis worker process , cpu usage, based on pick jobs , process them…..
if how can monitor iis worker process , cpu usage using c# codes……?

or other thoughts.....thanks.

update:
code snippet creating parallel threads:

public class parallelprocess   {     #region "instance constructor"     public parallelprocess()     {       //handle unhandled execeptions, throws threads.       appdomain.currentdomain.unhandledexception += onunhandledexception;     }     #endregion  #region "private fields"  #region "static" //number of threads activated process each job request. private static int maxconcurrentthreads = 0; //minimum number of idle threads should exist in pool. private static int threadbuffer = 0; //time out in ms theadpool wait on threads complete. private static int threadwaittimeout = 0; //available worker threads in thread pool private static int workerthreads = 0; //available port threads in thread pool private static int completionthreads = 0; //minimum worker threads available in threadpool private static int minw = 0; //minimum port threads available in threadpool #endregion private static int minc = 0;  #endregion  #region "static constructor"     static parallelprocess()     {       try       {         //required threads defined in c:\hobooappserver_config\hobooappserver_config.xml         maxconcurrentthreads = configuration.requiredthread;         threadbuffer =configuration.threadbuffer;         //in milliseconds         threadwaittimeout =configuration.trwaittime;          //in milliseconds           //get min number of threads thread pool.          threadpool.getminthreads(minw, minc);           //set thead limit spawn in current thread pool.          if (minw >= maxconcurrentthreads)          {           threadpool.setminthreads(maxconcurrentthreads, minc);           minw = maxconcurrentthreads;          }         else         {           threadpool.setminthreads(minw, minc);         }          }       catch (exception ex)       {         //write exception log file.         writejoblog(new joblogdto         {           mode = "parallel",           uniqueid = "thread pool exception",           threadid = thread.currentthread.managedthreadid.tostring(),           starttime = datetime.now.tostring(),           exceptionorresult = ex.tostring()         });        }     }     #endregion   #region "public methods" /// <summary> /// gets count of rows retrieved job. /// takes account current work load , available threads process. /// </summary> /// <returns>int (number of threads available)</returns> private int getmaxitemsretrievalcount() {   int rtnval = 1;   try   {     //get available idle threads in thead pool.     threadpool.getavailablethreads(workerthreads, completionthreads);     rtnval = workerthreads > maxconcurrentthreads ? maxconcurrentthreads : workerthreads;     rtnval = rtnval > 0 ? rtnval : 0;   }   catch (exception ex)   {     //write exceptions log file.     writejoblog(new joblogdto     {       mode = "parallel",       uniqueid = "getmaxitemsretrievalcount exception",       threadid = thread.currentthread.managedthreadid.tostring(),       starttime = datetime.now.tostring(),       exceptionorresult = ex.tostring()     });   }   return rtnval; }  /// <summary> /// method processes jobs on worker threads. /// </summary> public void processbatchjobs(bool pisnight, bool plpriority, string puniqueid) {   bool iscontinue = true;   int maxrecordcount = 0;   manualresetevent[] signalevents = null;      {     maxrecordcount = getmaxitemsretrievalcount();     if (maxrecordcount > 0)     {       //pick jobs database       list<snapshottask> jobs =business.rtds.snapshot.picktasks(pisnight, plpriority);       if (jobs != null && jobs.count > 0)       {         //log header-thread pool information , statistics - in parallel threads         writejoblog(new joblogdto         {           mode = "parallel",           uniqueid = puniqueid,           threadid = thread.currentthread.managedthreadid.tostring(),           starttime = datetime.now.tostring(),           avblworkerthread = workerthreads.tostring(),           avblportthread = completionthreads.tostring(),           acqrdworkerthread = minw.tostring(),           acqurdportthread = minc.tostring(),           jobstoprocess = jobs.count.tostring()         });          signalevents = new manualresetevent[jobs.count];         int signalcount = 0;          //loop through each job, create call function, add items queue, fire them         foreach (snapshottask job in jobs)         {           signalevents(signalcount) = new manualresetevent(false);           batchcallback threadpoolcallback =             new batchcallback(job, signalevents(signalcount));           bool nresult =             threadpool.queueuserworkitem(            new waitcallback(threadpoolcallback.threadpoolcallback),            new batchthreaddata           {             isnight = pisnight,             uniqueid = puniqueid           });           signalcount += 1;         }          //wait here untill child threads finish job or timeout meets.         bool result = waithandle.waitall(signalevents, parallelprocess.threadwaittimeout);          //when 1 or more threads have not set signal.         if (result == false)         {           //logger.write("not threads completed in pool.            //the pool exited due time-out.");         }         else         {           //logging time taken per batch.           //logger.write(string.format("average time taken each batch of {1} orders : {0} ms",           //new timespan((ticksend - ticksstart)).totalmilliseconds.tostring());         }         //ticksend = datetime.now.ticks       }       else       {         //todo : retry logic         //nothing process.         iscontinue = false;       }     }     else     {       //we did not thread execute. wait free thread(s).       thread.sleep(1000);     }   } while (iscontinue);    //end time after batch done.   //endtime = datetime.now   //log over-all time taken. } /// <summary> /// log unhandled exceptions application domain if any. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void onunhandledexception(object sender, unhandledexceptioneventargs e) {   exception ex = e.exceptionobject exception;   //write unhandled exception log file.   writejoblog(new joblogdto   {     mode = "parallel",     uniqueid = "unhandledexception",     threadid = thread.currentthread.managedthreadid.tostring(),     starttime = datetime.now.tostring(),     exceptionorresult = ex.tostring()   }); } #endregion } 

you not have monitor cpu load, os you. schedule threads depending on priority.

set thread of tasks lower priority. doing so, these tasks delayed until cpu free execute less important threads.

set priority this:

thread.currentthread.threadpriority = threadpriority.belownormal; // or threadpriority.lowest.