Empty output in asyncio web scraping

-1

I'am new to Python asyncio web scraping. I want to export scraped data to excel using pandas. Seems like my code bellow is working but while I am exporting output using pandas to excel then I am getting empty output file.

import asyncio

from concurrent.futures.thread import ThreadPoolExecutor

import pandas as pd

import os

import requests

from bs4 import BeautifulSoup



input_file = os.path.join(os.getcwd(), 'Sample.xlsx')

df = pd.read_excel(input_file, usecols=0)



req = requests.Session()

req.trust_env = False



Url, title, price = , , ,

executor = ThreadPoolExecutor(10)





def scrape(url, *, loop):

    loop.run_in_executor(executor, load_html, url)





def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    Url.append(url)

    title.append(soup.select('.pinfo-title')[0].text)

    price.append(soup.select('.sale-price')[0].text)





loop = asyncio.get_event_loop()

for url in df['Urls']:

    scrape(url, loop=loop)



loop.run_until_complete(asyncio.gather(*asyncio.all_tasks(loop)))



output = pd.DataFrame({

        'Url': Url,

        'Title': title,

        'Price': price

})

output.to_excel('Output.xlsx', index=False)

but If i use print instead of appending lists like bellow, then it's print desired output.

def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    print(url)

    print(soup.select('.pinfo-title')[0].text)

    print(soup.select('.sale-price')[0].text)

asked 4 mins ago

Atik

New contributor

add a comment |

-1

import asyncio

from concurrent.futures.thread import ThreadPoolExecutor

import pandas as pd

import os

import requests

from bs4 import BeautifulSoup



input_file = os.path.join(os.getcwd(), 'Sample.xlsx')

df = pd.read_excel(input_file, usecols=0)



req = requests.Session()

req.trust_env = False



Url, title, price = , , ,

executor = ThreadPoolExecutor(10)





def scrape(url, *, loop):

    loop.run_in_executor(executor, load_html, url)





def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    Url.append(url)

    title.append(soup.select('.pinfo-title')[0].text)

    price.append(soup.select('.sale-price')[0].text)





loop = asyncio.get_event_loop()

for url in df['Urls']:

    scrape(url, loop=loop)



loop.run_until_complete(asyncio.gather(*asyncio.all_tasks(loop)))



output = pd.DataFrame({

        'Url': Url,

        'Title': title,

        'Price': price

})

output.to_excel('Output.xlsx', index=False)

but If i use print instead of appending lists like bellow, then it's print desired output.

def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    print(url)

    print(soup.select('.pinfo-title')[0].text)

    print(soup.select('.sale-price')[0].text)

asked 4 mins ago

Atik

New contributor

add a comment |

-1

import asyncio

from concurrent.futures.thread import ThreadPoolExecutor

import pandas as pd

import os

import requests

from bs4 import BeautifulSoup



input_file = os.path.join(os.getcwd(), 'Sample.xlsx')

df = pd.read_excel(input_file, usecols=0)



req = requests.Session()

req.trust_env = False



Url, title, price = , , ,

executor = ThreadPoolExecutor(10)





def scrape(url, *, loop):

    loop.run_in_executor(executor, load_html, url)





def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    Url.append(url)

    title.append(soup.select('.pinfo-title')[0].text)

    price.append(soup.select('.sale-price')[0].text)





loop = asyncio.get_event_loop()

for url in df['Urls']:

    scrape(url, loop=loop)



loop.run_until_complete(asyncio.gather(*asyncio.all_tasks(loop)))



output = pd.DataFrame({

        'Url': Url,

        'Title': title,

        'Price': price

})

output.to_excel('Output.xlsx', index=False)

but If i use print instead of appending lists like bellow, then it's print desired output.

def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    print(url)

    print(soup.select('.pinfo-title')[0].text)

    print(soup.select('.sale-price')[0].text)

asked 4 mins ago

Atik

New contributor

import asyncio

from concurrent.futures.thread import ThreadPoolExecutor

import pandas as pd

import os

import requests

from bs4 import BeautifulSoup



input_file = os.path.join(os.getcwd(), 'Sample.xlsx')

df = pd.read_excel(input_file, usecols=0)



req = requests.Session()

req.trust_env = False



Url, title, price = , , ,

executor = ThreadPoolExecutor(10)





def scrape(url, *, loop):

    loop.run_in_executor(executor, load_html, url)





def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    Url.append(url)

    title.append(soup.select('.pinfo-title')[0].text)

    price.append(soup.select('.sale-price')[0].text)





loop = asyncio.get_event_loop()

for url in df['Urls']:

    scrape(url, loop=loop)



loop.run_until_complete(asyncio.gather(*asyncio.all_tasks(loop)))



output = pd.DataFrame({

        'Url': Url,

        'Title': title,

        'Price': price

})

output.to_excel('Output.xlsx', index=False)

but If i use print instead of appending lists like bellow, then it's print desired output.

def load_html(url):

    print(url)

    res = req.get(url)

    soup = BeautifulSoup(res.content, 'html.parser')

    print(url)

    print(soup.select('.pinfo-title')[0].text)

    print(soup.select('.sale-price')[0].text)

python web-scraping asynchronous pandas async-await

asked 4 mins ago

Atik

New contributor

asked 4 mins ago

Atik

New contributor

asked 4 mins ago

Atik

New contributor

asked 4 mins ago

Atik

asked 4 mins ago

Atik

New contributor

Atik is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.

add a comment |

active

oldest

votes

Your Answer

StackExchange.ifUsing("editor", function () {
return StackExchange.using("mathjaxEditing", function () {
StackExchange.MarkdownEditor.creationCallbacks.add(function (editor, postfix) {
StackExchange.mathjaxEditing.prepareWmdForMathJax(editor, postfix, [["\$", "\$"]]);
});
});
}, "mathjax-editing");

StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");

StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "196"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);

StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});

function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: false,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: null,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});

}
});

Atik is a new contributor. Be nice, and check out our Code of Conduct.

draft saved

draft discarded

Sign up or log in

StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});

Post as a guest

Name

Required, but never shown

StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fcodereview.stackexchange.com%2fquestions%2f210312%2fempty-output-in-asyncio-web-scraping%23new-answer', 'question_page');
}
);

Post as a guest

Name

Required, but never shown

active

oldest

votes

Atik is a new contributor. Be nice, and check out our Code of Conduct.

draft saved

draft discarded

Atik is a new contributor. Be nice, and check out our Code of Conduct.

Thanks for contributing an answer to Code Review Stack Exchange!

Please be sure to answer the question. Provide details and share your research!

But avoid …

Asking for help, clarification, or responding to other answers.

Making statements based on opinion; back them up with references or personal experience.

Use MathJax to format equations. MathJax reference.

To learn more, see our tips on writing great answers.

Some of your past answers have not been well-received, and you're in danger of being blocked from answering.

Please pay close attention to the following guidance:

Please be sure to answer the question. Provide details and share your research!

But avoid …

Asking for help, clarification, or responding to other answers.

Making statements based on opinion; back them up with references or personal experience.

To learn more, see our tips on writing great answers.

draft saved

draft discarded

Sign up or log in

StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});

Post as a guest

Name

Required, but never shown

Post as a guest

Name

Required, but never shown

Sign up or log in

StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});

Post as a guest

Name

Required, but never shown

Sign up or log in

StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});

Post as a guest

Name

Required, but never shown

Sign up or log in

StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});

Post as a guest

Name

Required, but never shown

Name

Required, but never shown

Name

Required, but never shown

This page is only for reference, If you need detailed information, please check here

搜尋此網誌

Tukukkk