@techreport{ilprints505,
          number = {2001-35},
          author = {Junghoo Cho and Hector Garcia-Molina},
           title = {Parallel Crawlers},
            type = {Technical Report},
       publisher = {Stanford},
     institution = {Stanford InfoLab},
         journal = {Technical report},
            year = {2001},
        keywords = {Web crawler, parallalism, distributed crawler},
             url = {http://ilpubs.stanford.edu:8090/505/},
        abstract = {In this paper we study how we can design an effective parallel crawler. As the size of the Web grows, it becomes imperative to parallelize a crawling process, in order to finish downloading pages in a reasonable amount of time.  We first propose multiple architectures for a parallel crawler and identify fundamental issues related to parallel crawling. Based on this understanding, we then propose metrics to evaluate a parallel crawler, and compare the proposed architectures using 40 million pages collected from the Web. Our results clarify the relative merits of each architecture and provide a good guideline on when to adopt which architecture.}
}