Modification in web scraping scenarios

hjsblogger · hjsblogger · commit 1a817bc818dd · 2024-02-26T15:35:59.000+05:30
diff --git a/tests/web-scraping/test_async_ecommerce_scrapping.py b/tests/web-scraping/test_async_ecommerce_scrapping.py
@@ -3,10 +3,11 @@
 import sys
 import ssl
 import certifi
+import time
 from pprint import pprint
 from bs4 import BeautifulSoup
 
-sys.path.append(sys.path[0] + "/../../..")
+sys.path.append(sys.path[0] + "/../../")
 
 from pageobject.locators import locators
 from pageobject.helpers import helpers
@@ -25,7 +26,7 @@ async def fetch(url, session):
 
 async def scrap_ecommerce(url):
     ssl_context = ssl.create_default_context(cafile=certifi.where())
-    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl_context=ssl_context)) as session:
+    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session:
         html = await fetch(url, session)
         soup = BeautifulSoup(html, 'html.parser')
 
@@ -48,6 +49,7 @@ async def scrap_ecommerce(url):
         return meta_data_arr
 
 async def main():
+    start_time = time.time()
     base_url = locators.test_bs4_url
     tasks = [scrap_ecommerce(f"{base_url}&page={i}") for i in range(1, 6)]
     results = await asyncio.gather(*tasks)
@@ -57,7 +59,8 @@ async def main():
         print("*********************************************************************************************************")
         helpers.print_scrapped_content(result)
         print()
+    
+    print("\nTime elapsed is " + str((time.time() - start_time)) + " seconds")
 
 if __name__ == '__main__':
-    output = asyncio.run(main())
-    print(output)
+    output = asyncio.run(main())
diff --git a/tests/web-scraping/test_sync_ecommerce_scrapping.py b/tests/web-scraping/test_sync_ecommerce_scrapping.py
@@ -3,7 +3,7 @@
 # Import the locators file
 import sys
 from pprint import pprint
-sys.path.append(sys.path[0] + "/../../..")
+sys.path.append(sys.path[0] + "/../../")
 
 from pageobject.locators import locators
 from pageobject.locators import *
@@ -44,10 +44,13 @@ def scrap_ecommerce(url) -> list:
 # Page 1: https://ecommerce-playground.lambdatest.io/index.php?route=product/category&path=57&page=1
 # Page 5: https://ecommerce-playground.lambdatest.io/index.php?route=product/category&path=57&page=5
 if __name__ == '__main__':
+    start_time = time.time()
     for iteration in range(1,6):
         test_url = locators.test_bs4_url + "&page=" + str(iteration)
         meta_data_arr = scrap_ecommerce(test_url)
         print('\n')
         print("Product Page = " + test_url)
         print("*********************************************************************************************************\n")
-        helpers.print_scrapped_content(meta_data_arr)
+        helpers.print_scrapped_content(meta_data_arr)
+    
+        print("\nTime elapsed is " + str((time.time() - start_time)) + " seconds")
