first commit
This commit is contained in:
govardhan
112
sample_output/parallel_scraper.log
Normal file
112
sample_output/parallel_scraper.log
Normal file
@ -0,0 +1,112 @@
|
||||
2025-08-18 22:50:54,088 - INFO - Starting parallel scraping from page 1 to 100
|
||||
2025-08-18 22:50:54,089 - INFO - Batch size: 20, Max workers: 5
|
||||
2025-08-18 22:50:54,089 - INFO - Scrape details: False
|
||||
2025-08-18 22:50:54,089 - INFO - Created 5 batches
|
||||
2025-08-18 22:50:54,089 - INFO - Starting batch processing...
|
||||
2025-08-18 22:50:55,608 - INFO - Page 1: Found 30 companies
|
||||
2025-08-18 22:50:55,630 - INFO - Page 3: Found 30 companies
|
||||
2025-08-18 22:50:55,941 - INFO - Page 4: Found 30 companies
|
||||
2025-08-18 22:50:56,013 - INFO - Page 2: Found 30 companies
|
||||
2025-08-18 22:50:56,033 - INFO - Page 5: Found 30 companies
|
||||
2025-08-18 22:50:56,575 - INFO - Page 21: Found 30 companies
|
||||
2025-08-18 22:50:56,656 - INFO - Page 22: Found 30 companies
|
||||
2025-08-18 22:50:57,019 - INFO - Page 23: Found 30 companies
|
||||
2025-08-18 22:50:57,118 - INFO - Page 24: Found 30 companies
|
||||
2025-08-18 22:50:57,156 - INFO - Page 25: Found 30 companies
|
||||
2025-08-18 22:50:57,622 - INFO - Page 42: Found 30 companies
|
||||
2025-08-18 22:50:57,697 - INFO - Page 41: Found 30 companies
|
||||
2025-08-18 22:50:58,020 - INFO - Page 43: Found 30 companies
|
||||
2025-08-18 22:50:58,283 - INFO - Page 45: Found 30 companies
|
||||
2025-08-18 22:50:58,323 - INFO - Page 44: Found 30 companies
|
||||
2025-08-18 22:50:58,702 - INFO - Page 61: Found 30 companies
|
||||
2025-08-18 22:50:58,831 - INFO - Page 62: Found 30 companies
|
||||
2025-08-18 22:50:59,097 - INFO - Page 63: Found 30 companies
|
||||
2025-08-18 22:50:59,342 - INFO - Page 64: Found 30 companies
|
||||
2025-08-18 22:50:59,461 - INFO - Page 49: Found 30 companies
|
||||
2025-08-18 22:50:59,740 - INFO - Page 65: Found 30 companies
|
||||
2025-08-18 22:50:59,791 - INFO - Page 81: Found 30 companies
|
||||
2025-08-18 22:51:00,093 - INFO - Page 82: Found 30 companies
|
||||
2025-08-18 22:51:00,467 - INFO - Page 83: Found 30 companies
|
||||
2025-08-18 22:51:00,578 - INFO - Page 84: Found 30 companies
|
||||
2025-08-18 22:51:00,772 - INFO - Page 85: Found 30 companies
|
||||
2025-08-18 22:51:00,793 - INFO - Page 6: Found 30 companies
|
||||
2025-08-18 22:51:01,193 - INFO - Page 7: Found 30 companies
|
||||
2025-08-18 22:51:01,563 - INFO - Page 8: Found 30 companies
|
||||
2025-08-18 22:51:01,710 - INFO - Page 9: Found 30 companies
|
||||
2025-08-18 22:51:01,771 - INFO - Page 26: Found 30 companies
|
||||
2025-08-18 22:51:01,793 - INFO - Page 10: Found 30 companies
|
||||
2025-08-18 22:51:02,265 - INFO - Page 27: Found 30 companies
|
||||
2025-08-18 22:51:02,699 - INFO - Page 28: Found 30 companies
|
||||
2025-08-18 22:51:02,829 - INFO - Page 46: Found 30 companies
|
||||
2025-08-18 22:51:02,882 - INFO - Page 29: Found 30 companies
|
||||
2025-08-18 22:51:03,237 - INFO - Page 30: Found 30 companies
|
||||
2025-08-18 22:51:03,346 - INFO - Page 47: Found 30 companies
|
||||
2025-08-18 22:51:03,808 - INFO - Page 48: Found 30 companies
|
||||
2025-08-18 22:51:03,873 - INFO - Page 50: Found 30 companies
|
||||
2025-08-18 22:51:04,108 - INFO - Page 66: Found 30 companies
|
||||
2025-08-18 22:51:04,190 - INFO - Page 67: Found 30 companies
|
||||
2025-08-18 22:51:04,415 - INFO - Page 68: Found 30 companies
|
||||
2025-08-18 22:51:04,900 - INFO - Page 51: Found 30 companies
|
||||
2025-08-18 22:51:04,954 - INFO - Page 69: Found 30 companies
|
||||
2025-08-18 22:51:05,137 - INFO - Page 86: Found 30 companies
|
||||
2025-08-18 22:51:05,270 - INFO - Page 70: Found 30 companies
|
||||
2025-08-18 22:51:05,524 - INFO - Page 87: Found 30 companies
|
||||
2025-08-18 22:51:05,972 - INFO - Page 88: Found 30 companies
|
||||
2025-08-18 22:51:06,091 - INFO - Page 89: Found 30 companies
|
||||
2025-08-18 22:51:06,111 - INFO - Page 90: Found 30 companies
|
||||
2025-08-18 22:51:06,438 - INFO - Page 11: Found 30 companies
|
||||
2025-08-18 22:51:06,644 - INFO - Page 12: Found 30 companies
|
||||
2025-08-18 22:51:06,990 - INFO - Page 13: Found 30 companies
|
||||
2025-08-18 22:51:07,076 - INFO - Page 31: Found 30 companies
|
||||
2025-08-18 22:51:07,208 - INFO - Page 14: Found 30 companies
|
||||
2025-08-18 22:51:07,520 - INFO - Page 15: Found 30 companies
|
||||
2025-08-18 22:51:07,747 - INFO - Page 32: Found 30 companies
|
||||
2025-08-18 22:51:08,076 - INFO - Page 33: Found 30 companies
|
||||
2025-08-18 22:51:08,097 - INFO - Page 52: Found 30 companies
|
||||
2025-08-18 22:51:08,356 - INFO - Page 34: Found 30 companies
|
||||
2025-08-18 22:51:08,608 - INFO - Page 35: Found 30 companies
|
||||
2025-08-18 22:51:08,817 - INFO - Page 53: Found 30 companies
|
||||
2025-08-18 22:51:09,073 - INFO - Page 55: Found 30 companies
|
||||
2025-08-18 22:51:09,115 - INFO - Page 54: Found 30 companies
|
||||
2025-08-18 22:51:09,491 - INFO - Page 71: Found 30 companies
|
||||
2025-08-18 22:51:09,724 - INFO - Page 72: Found 30 companies
|
||||
2025-08-18 22:51:09,955 - INFO - Page 73: Found 30 companies
|
||||
2025-08-18 22:51:10,037 - INFO - Page 56: Found 30 companies
|
||||
2025-08-18 22:51:10,190 - INFO - Page 74: Found 30 companies
|
||||
2025-08-18 22:51:10,656 - INFO - Page 91: Found 30 companies
|
||||
2025-08-18 22:51:10,818 - INFO - Page 75: Found 30 companies
|
||||
2025-08-18 22:51:11,025 - INFO - Page 92: Found 30 companies
|
||||
2025-08-18 22:51:11,046 - INFO - Page 93: Found 30 companies
|
||||
2025-08-18 22:51:11,248 - INFO - Page 94: Found 30 companies
|
||||
2025-08-18 22:51:11,758 - INFO - Page 95: Found 30 companies
|
||||
2025-08-18 22:51:11,921 - INFO - Page 16: Found 30 companies
|
||||
2025-08-18 22:51:12,031 - INFO - Page 18: Found 30 companies
|
||||
2025-08-18 22:51:12,126 - INFO - Page 17: Found 30 companies
|
||||
2025-08-18 22:51:12,465 - INFO - Page 36: Found 30 companies
|
||||
2025-08-18 22:51:12,911 - INFO - Page 19: Found 30 companies
|
||||
2025-08-18 22:51:12,977 - INFO - Page 37: Found 30 companies
|
||||
2025-08-18 22:51:13,058 - INFO - Page 20: Found 30 companies
|
||||
2025-08-18 22:51:13,066 - INFO - Saved batch 1 with 600 companies
|
||||
2025-08-18 22:51:13,219 - INFO - Page 38: Found 30 companies
|
||||
2025-08-18 22:51:13,484 - INFO - Page 57: Found 30 companies
|
||||
2025-08-18 22:51:13,922 - INFO - Page 40: Found 30 companies
|
||||
2025-08-18 22:51:14,030 - INFO - Page 39: Found 30 companies
|
||||
2025-08-18 22:51:14,036 - INFO - Saved batch 2 with 600 companies
|
||||
2025-08-18 22:51:14,256 - INFO - Page 58: Found 30 companies
|
||||
2025-08-18 22:51:14,281 - INFO - Page 59: Found 30 companies
|
||||
2025-08-18 22:51:14,466 - INFO - Page 60: Found 30 companies
|
||||
2025-08-18 22:51:14,472 - INFO - Saved batch 3 with 600 companies
|
||||
2025-08-18 22:51:14,869 - INFO - Page 76: Found 30 companies
|
||||
2025-08-18 22:51:15,077 - INFO - Page 77: Found 30 companies
|
||||
2025-08-18 22:51:15,339 - INFO - Page 78: Found 30 companies
|
||||
2025-08-18 22:51:15,363 - INFO - Page 79: Found 30 companies
|
||||
2025-08-18 22:51:15,496 - INFO - Page 96: Found 30 companies
|
||||
2025-08-18 22:51:15,841 - INFO - Page 80: Found 30 companies
|
||||
2025-08-18 22:51:15,848 - INFO - Saved batch 4 with 600 companies
|
||||
2025-08-18 22:51:16,151 - INFO - Page 97: Found 30 companies
|
||||
2025-08-18 22:51:16,448 - INFO - Page 99: Found 30 companies
|
||||
2025-08-18 22:51:16,473 - INFO - Page 98: Found 30 companies
|
||||
2025-08-18 22:51:16,549 - INFO - Page 100: Found 30 companies
|
||||
2025-08-18 22:51:16,554 - INFO - Saved batch 5 with 600 companies
|
||||
2025-08-18 22:51:16,554 - INFO - Completed: 5 successful, 0 failed batches
|
||||
2025-08-18 22:51:16,577 - INFO - Saved 3000 companies to final files
|
||||
Reference in New Issue
Block a user