Constructor function isn't get call after a specific wait - Javascript
I am working on a scrapper, where I scrap jobs from a website. First I pass the the url of that website to the member function of that constructor, when It loads the page, after that I evaluate the number of jobs and number of pagination pages in the website keep them in the variables jobs and pages. When the site isn't fully loaded, the value of jobs and pages are null.
If the values are null, then I recall the waitMore() function inside the load() function, which waits for 5 more seconds until the page is loaded, and then the condition is checked if the jobs and pages values are still null. Once this condition is false then the values of jobs and pages are returned.
The reason of recalling the waitMore() function is that, sometimes the internet is slow and the page is not fully loaded so I just wait for the page to load fully. But when the internet is faster I get the values perfectly, but when Its slower the waitMore() is called again and again, until the condition is set to false i.e; when the jobs and pages will have values. When they have the values, then the values are not returned. I don't know why.
My Code
function Scrap() {
this.loadPage = async function(url, loadTime) {
ph_instance = await phantom.create();
ph_page = await ph_instance.createPage();
var status = await ph_page.open(url); // Load the Pages
if (status == 'success') {
async function waitMore() {
let p = new Promise(function(res, rej) {
setTimeout(async function() {
var pages, jobs;
jobs = await someFunction1();
pages = await someFunction2();
if (jobs === '' || pages === '' || jobs === 'undefined' || pages === 'undefined' || jobs === null || pages === null) {
waitMore();
} else {
res({
success: true,
data: {
"jobs": jobs,
"pages": pages
}
});
}
}, loadTime);
});
var res = await p;
return res;
}
return waitMore();
};
}
}
new Scrap().loadPage("example.com", 5000).then(function(res) {
console.log("Response ", res);
})javascript node.js
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
add a comment |
I am working on a scrapper, where I scrap jobs from a website. First I pass the the url of that website to the member function of that constructor, when It loads the page, after that I evaluate the number of jobs and number of pagination pages in the website keep them in the variables jobs and pages. When the site isn't fully loaded, the value of jobs and pages are null.
If the values are null, then I recall the waitMore() function inside the load() function, which waits for 5 more seconds until the page is loaded, and then the condition is checked if the jobs and pages values are still null. Once this condition is false then the values of jobs and pages are returned.
The reason of recalling the waitMore() function is that, sometimes the internet is slow and the page is not fully loaded so I just wait for the page to load fully. But when the internet is faster I get the values perfectly, but when Its slower the waitMore() is called again and again, until the condition is set to false i.e; when the jobs and pages will have values. When they have the values, then the values are not returned. I don't know why.
My Code
function Scrap() {
this.loadPage = async function(url, loadTime) {
ph_instance = await phantom.create();
ph_page = await ph_instance.createPage();
var status = await ph_page.open(url); // Load the Pages
if (status == 'success') {
async function waitMore() {
let p = new Promise(function(res, rej) {
setTimeout(async function() {
var pages, jobs;
jobs = await someFunction1();
pages = await someFunction2();
if (jobs === '' || pages === '' || jobs === 'undefined' || pages === 'undefined' || jobs === null || pages === null) {
waitMore();
} else {
res({
success: true,
data: {
"jobs": jobs,
"pages": pages
}
});
}
}, loadTime);
});
var res = await p;
return res;
}
return waitMore();
};
}
}
new Scrap().loadPage("example.com", 5000).then(function(res) {
console.log("Response ", res);
})javascript node.js
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
add a comment |
I am working on a scrapper, where I scrap jobs from a website. First I pass the the url of that website to the member function of that constructor, when It loads the page, after that I evaluate the number of jobs and number of pagination pages in the website keep them in the variables jobs and pages. When the site isn't fully loaded, the value of jobs and pages are null.
If the values are null, then I recall the waitMore() function inside the load() function, which waits for 5 more seconds until the page is loaded, and then the condition is checked if the jobs and pages values are still null. Once this condition is false then the values of jobs and pages are returned.
The reason of recalling the waitMore() function is that, sometimes the internet is slow and the page is not fully loaded so I just wait for the page to load fully. But when the internet is faster I get the values perfectly, but when Its slower the waitMore() is called again and again, until the condition is set to false i.e; when the jobs and pages will have values. When they have the values, then the values are not returned. I don't know why.
My Code
function Scrap() {
this.loadPage = async function(url, loadTime) {
ph_instance = await phantom.create();
ph_page = await ph_instance.createPage();
var status = await ph_page.open(url); // Load the Pages
if (status == 'success') {
async function waitMore() {
let p = new Promise(function(res, rej) {
setTimeout(async function() {
var pages, jobs;
jobs = await someFunction1();
pages = await someFunction2();
if (jobs === '' || pages === '' || jobs === 'undefined' || pages === 'undefined' || jobs === null || pages === null) {
waitMore();
} else {
res({
success: true,
data: {
"jobs": jobs,
"pages": pages
}
});
}
}, loadTime);
});
var res = await p;
return res;
}
return waitMore();
};
}
}
new Scrap().loadPage("example.com", 5000).then(function(res) {
console.log("Response ", res);
})javascript node.js
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
I am working on a scrapper, where I scrap jobs from a website. First I pass the the url of that website to the member function of that constructor, when It loads the page, after that I evaluate the number of jobs and number of pagination pages in the website keep them in the variables jobs and pages. When the site isn't fully loaded, the value of jobs and pages are null.
If the values are null, then I recall the waitMore() function inside the load() function, which waits for 5 more seconds until the page is loaded, and then the condition is checked if the jobs and pages values are still null. Once this condition is false then the values of jobs and pages are returned.
The reason of recalling the waitMore() function is that, sometimes the internet is slow and the page is not fully loaded so I just wait for the page to load fully. But when the internet is faster I get the values perfectly, but when Its slower the waitMore() is called again and again, until the condition is set to false i.e; when the jobs and pages will have values. When they have the values, then the values are not returned. I don't know why.
My Code
function Scrap() {
this.loadPage = async function(url, loadTime) {
ph_instance = await phantom.create();
ph_page = await ph_instance.createPage();
var status = await ph_page.open(url); // Load the Pages
if (status == 'success') {
async function waitMore() {
let p = new Promise(function(res, rej) {
setTimeout(async function() {
var pages, jobs;
jobs = await someFunction1();
pages = await someFunction2();
if (jobs === '' || pages === '' || jobs === 'undefined' || pages === 'undefined' || jobs === null || pages === null) {
waitMore();
} else {
res({
success: true,
data: {
"jobs": jobs,
"pages": pages
}
});
}
}, loadTime);
});
var res = await p;
return res;
}
return waitMore();
};
}
}
new Scrap().loadPage("example.com", 5000).then(function(res) {
console.log("Response ", res);
})function Scrap() {
this.loadPage = async function(url, loadTime) {
ph_instance = await phantom.create();
ph_page = await ph_instance.createPage();
var status = await ph_page.open(url); // Load the Pages
if (status == 'success') {
async function waitMore() {
let p = new Promise(function(res, rej) {
setTimeout(async function() {
var pages, jobs;
jobs = await someFunction1();
pages = await someFunction2();
if (jobs === '' || pages === '' || jobs === 'undefined' || pages === 'undefined' || jobs === null || pages === null) {
waitMore();
} else {
res({
success: true,
data: {
"jobs": jobs,
"pages": pages
}
});
}
}, loadTime);
});
var res = await p;
return res;
}
return waitMore();
};
}
}
new Scrap().loadPage("example.com", 5000).then(function(res) {
console.log("Response ", res);
})function Scrap() {
this.loadPage = async function(url, loadTime) {
ph_instance = await phantom.create();
ph_page = await ph_instance.createPage();
var status = await ph_page.open(url); // Load the Pages
if (status == 'success') {
async function waitMore() {
let p = new Promise(function(res, rej) {
setTimeout(async function() {
var pages, jobs;
jobs = await someFunction1();
pages = await someFunction2();
if (jobs === '' || pages === '' || jobs === 'undefined' || pages === 'undefined' || jobs === null || pages === null) {
waitMore();
} else {
res({
success: true,
data: {
"jobs": jobs,
"pages": pages
}
});
}
}, loadTime);
});
var res = await p;
return res;
}
return waitMore();
};
}
}
new Scrap().loadPage("example.com", 5000).then(function(res) {
console.log("Response ", res);
})javascript node.js
javascript node.js
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
asked Nov 22 '18 at 10:28
Nadeem AhmadNadeem Ahmad
838
838
add a comment |
add a comment |
0
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53428862%2fconstructor-function-isnt-get-call-after-a-specific-wait-javascript%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
0
active
oldest
votes
0
active
oldest
votes
active
oldest
votes
active
oldest
votes
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53428862%2fconstructor-function-isnt-get-call-after-a-specific-wait-javascript%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
