"asyncio.run() cannot be called from a running event loop" when using Jupyter Notebook
Question:
I would like to use asyncio to get webpage html.
I run the following code in jupyter notebook:
import aiofiles
import aiohttp
from aiohttp import ClientSession
async def get_info(url, session):
resp = await session.request(method="GET", url=url)
resp.raise_for_status()
html = await resp.text(encoding='GB18030')
with open('test_asyncio.html', 'w', encoding='utf-8-sig') as f:
f.write(html)
return html
async def main(urls):
async with ClientSession() as session:
tasks = [get_info(url, session) for url in urls]
return await asyncio.gather(*tasks)
if __name__ == "__main__":
url = ['http://huanyuntianxiazh.fang.com/house/1010123799/housedetail.htm', 'http://zhaoshangyonghefu010.fang.com/house/1010126863/housedetail.htm']
result = asyncio.run(main(url))
However, it returns RuntimeError: asyncio.run() cannot be called from a running event loop
What is the problem?
How to solve it?
Answers:
The asyncio.run()
documentation says:
This function cannot be called when another asyncio event loop is running in the same thread.
In your case, jupyter (IPython ≥ 7.0) is already running an event loop:
You can now use async/await at the top level in the IPython terminal and in the notebook, it should — in most of the cases — “just work”. Update IPython to version 7+, IPykernel to version 5+, and you’re off to the races.
Therefore you don’t need to start the event loop yourself and can instead call await main(url)
directly, even if your code lies outside any asynchronous function.
Jupyter (IPython ≥ 7.0)
async def main():
print(1)
await main()
Python ≥ 3.7 and IPython < 7.0
import asyncio
async def main():
print(1)
asyncio.run(main())
In your code that would give:
url = ['url1', 'url2']
result = await main(url)
for text in result:
pass # text contains your html (text) response
Caution
There is a slight difference on how Jupyter uses the loop compared to IPython.
To add to cglacet
‘s answer – if one wants to detect whether a loop is running and adjust automatically (ie run main()
on the existing loop, otherwise asyncio.run()
), here is a snippet that may prove useful:
# async def main():
# ...
try:
loop = asyncio.get_running_loop()
except RuntimeError: # 'RuntimeError: There is no current event loop...'
loop = None
if loop and loop.is_running():
print('Async event loop already running. Adding coroutine to the event loop.')
tsk = loop.create_task(main())
# ^-- https://docs.python.org/3/library/asyncio-task.html#task-object
# Optionally, a callback function can be executed when the coroutine completes
tsk.add_done_callback(
lambda t: print(f'Task done with result={t.result()} << return val of main()'))
else:
print('Starting new event loop')
result = asyncio.run(main())
As cglacet mentioned that documentation says
This function cannot be called when another asyncio event loop is
running in the same thread.
You can use another thread i.e –
class ResolveThread(threading.Thread):
def __init__(self,result1,fun,url):
self.result1= result1
self.fun = fun
self.url = url
threading.Thread.__init__(self)
def run(self):
result1[0] = asyncio.run(self.fun(self.url))
result1 = [None]
sp = ResolveThread(result1)
sp.start()
sp.join() # connect main thread
result = result1[0]
Combining the methods from Pankaj Sharma and Jean Monet, I wrote the following snippet that acts as asyncio.run (with slightly different syntax), but also works within a Jupyter notebook.
class RunThread(threading.Thread):
def __init__(self, func, args, kwargs):
self.func = func
self.args = args
self.kwargs = kwargs
self.result = None
super().__init__()
def run(self):
self.result = asyncio.run(self.func(*self.args, **self.kwargs))
def run_async(func, *args, **kwargs):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop and loop.is_running():
thread = RunThread(func, args, kwargs)
thread.start()
thread.join()
return thread.result
else:
return asyncio.run(func(*args, **kwargs))
Usage:
async def test(name):
await asyncio.sleep(5)
return f"hello {name}"
run_async(test, "user") # blocks for 5 seconds and returns "hello user"
I found the unsync
package useful for writing code that behaves the same way in a Python script and the Jupyter REPL.
import asyncio
from unsync import unsync
@unsync
async def demo_async_fn():
await asyncio.sleep(0.1)
return "done!"
print(demo_async_fn().result())
Slight simplification of the solution by Mark:
import threading
class RunThread(threading.Thread):
def __init__(self, coro):
self.coro = coro
self.result = None
super().__init__()
def run(self):
self.result = asyncio.run(self.coro)
def run_async(coro):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop and loop.is_running():
thread = RunThread(coro)
thread.start()
thread.join()
return thread.result
else:
return asyncio.run(coro)
Use run_async()
like async.run()
, i.e., run_async(test("user"))
.
I would like to use asyncio to get webpage html.
I run the following code in jupyter notebook:
import aiofiles
import aiohttp
from aiohttp import ClientSession
async def get_info(url, session):
resp = await session.request(method="GET", url=url)
resp.raise_for_status()
html = await resp.text(encoding='GB18030')
with open('test_asyncio.html', 'w', encoding='utf-8-sig') as f:
f.write(html)
return html
async def main(urls):
async with ClientSession() as session:
tasks = [get_info(url, session) for url in urls]
return await asyncio.gather(*tasks)
if __name__ == "__main__":
url = ['http://huanyuntianxiazh.fang.com/house/1010123799/housedetail.htm', 'http://zhaoshangyonghefu010.fang.com/house/1010126863/housedetail.htm']
result = asyncio.run(main(url))
However, it returns RuntimeError: asyncio.run() cannot be called from a running event loop
What is the problem?
How to solve it?
The asyncio.run()
documentation says:
This function cannot be called when another asyncio event loop is running in the same thread.
In your case, jupyter (IPython ≥ 7.0) is already running an event loop:
You can now use async/await at the top level in the IPython terminal and in the notebook, it should — in most of the cases — “just work”. Update IPython to version 7+, IPykernel to version 5+, and you’re off to the races.
Therefore you don’t need to start the event loop yourself and can instead call await main(url)
directly, even if your code lies outside any asynchronous function.
Jupyter (IPython ≥ 7.0)
async def main():
print(1)
await main()
Python ≥ 3.7 and IPython < 7.0
import asyncio
async def main():
print(1)
asyncio.run(main())
In your code that would give:
url = ['url1', 'url2']
result = await main(url)
for text in result:
pass # text contains your html (text) response
Caution
There is a slight difference on how Jupyter uses the loop compared to IPython.
To add to cglacet
‘s answer – if one wants to detect whether a loop is running and adjust automatically (ie run main()
on the existing loop, otherwise asyncio.run()
), here is a snippet that may prove useful:
# async def main():
# ...
try:
loop = asyncio.get_running_loop()
except RuntimeError: # 'RuntimeError: There is no current event loop...'
loop = None
if loop and loop.is_running():
print('Async event loop already running. Adding coroutine to the event loop.')
tsk = loop.create_task(main())
# ^-- https://docs.python.org/3/library/asyncio-task.html#task-object
# Optionally, a callback function can be executed when the coroutine completes
tsk.add_done_callback(
lambda t: print(f'Task done with result={t.result()} << return val of main()'))
else:
print('Starting new event loop')
result = asyncio.run(main())
As cglacet mentioned that documentation says
This function cannot be called when another asyncio event loop is
running in the same thread.
You can use another thread i.e –
class ResolveThread(threading.Thread):
def __init__(self,result1,fun,url):
self.result1= result1
self.fun = fun
self.url = url
threading.Thread.__init__(self)
def run(self):
result1[0] = asyncio.run(self.fun(self.url))
result1 = [None]
sp = ResolveThread(result1)
sp.start()
sp.join() # connect main thread
result = result1[0]
Combining the methods from Pankaj Sharma and Jean Monet, I wrote the following snippet that acts as asyncio.run (with slightly different syntax), but also works within a Jupyter notebook.
class RunThread(threading.Thread):
def __init__(self, func, args, kwargs):
self.func = func
self.args = args
self.kwargs = kwargs
self.result = None
super().__init__()
def run(self):
self.result = asyncio.run(self.func(*self.args, **self.kwargs))
def run_async(func, *args, **kwargs):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop and loop.is_running():
thread = RunThread(func, args, kwargs)
thread.start()
thread.join()
return thread.result
else:
return asyncio.run(func(*args, **kwargs))
Usage:
async def test(name):
await asyncio.sleep(5)
return f"hello {name}"
run_async(test, "user") # blocks for 5 seconds and returns "hello user"
I found the unsync
package useful for writing code that behaves the same way in a Python script and the Jupyter REPL.
import asyncio
from unsync import unsync
@unsync
async def demo_async_fn():
await asyncio.sleep(0.1)
return "done!"
print(demo_async_fn().result())
Slight simplification of the solution by Mark:
import threading
class RunThread(threading.Thread):
def __init__(self, coro):
self.coro = coro
self.result = None
super().__init__()
def run(self):
self.result = asyncio.run(self.coro)
def run_async(coro):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop and loop.is_running():
thread = RunThread(coro)
thread.start()
thread.join()
return thread.result
else:
return asyncio.run(coro)
Use run_async()
like async.run()
, i.e., run_async(test("user"))
.