Trying to scrap javascript with PyQt4
0
I'm having trouble scraping a horse racing site, it seems it's reading the page too fast, and just returns "loading .... "
Here is my code:
import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs
import urllib.request
class Client(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self.on_page_load)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def on_page_load(self):
self.app.quit()
url = 'https://www.racenet.com.au/racing-form-guide/ballina-20181126/tab-com-au-maiden-plate-race-2/form'
client_response = Client(url)
source = client_response.mainFrame().toHtml()
soup = bs.BeautifulSoup(source, 'lxml')
js_test = soup.find('table', class_='form-table col-md-12')
print(js_test.text)
Output:
Loading....
python pyqt pyqt4 screen-scraping
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
asked Nov 26 '18 at 3:12
user3170725user3170725
377
add a comment |
0
I'm having trouble scraping a horse racing site, it seems it's reading the page too fast, and just returns "loading .... "
Here is my code:
import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs
import urllib.request
class Client(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self.on_page_load)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def on_page_load(self):
self.app.quit()
url = 'https://www.racenet.com.au/racing-form-guide/ballina-20181126/tab-com-au-maiden-plate-race-2/form'
client_response = Client(url)
source = client_response.mainFrame().toHtml()
soup = bs.BeautifulSoup(source, 'lxml')
js_test = soup.find('table', class_='form-table col-md-12')
print(js_test.text)
Output:
Loading....
python pyqt pyqt4 screen-scraping
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
asked Nov 26 '18 at 3:12
user3170725user3170725
377
add a comment |
0
0
0
I'm having trouble scraping a horse racing site, it seems it's reading the page too fast, and just returns "loading .... "
Here is my code:
import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs
import urllib.request
class Client(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self.on_page_load)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def on_page_load(self):
self.app.quit()
url = 'https://www.racenet.com.au/racing-form-guide/ballina-20181126/tab-com-au-maiden-plate-race-2/form'
client_response = Client(url)
source = client_response.mainFrame().toHtml()
soup = bs.BeautifulSoup(source, 'lxml')
js_test = soup.find('table', class_='form-table col-md-12')
print(js_test.text)
Output:
Loading....
python pyqt pyqt4 screen-scraping
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
asked Nov 26 '18 at 3:12
user3170725user3170725
377
I'm having trouble scraping a horse racing site, it seems it's reading the page too fast, and just returns "loading .... "
Here is my code:
import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebPage
import bs4 as bs
import urllib.request
class Client(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self.on_page_load)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def on_page_load(self):
self.app.quit()
url = 'https://www.racenet.com.au/racing-form-guide/ballina-20181126/tab-com-au-maiden-plate-race-2/form'
client_response = Client(url)
source = client_response.mainFrame().toHtml()
soup = bs.BeautifulSoup(source, 'lxml')
js_test = soup.find('table', class_='form-table col-md-12')
print(js_test.text)
Output:
Loading....
python pyqt pyqt4 screen-scraping
python pyqt pyqt4 screen-scraping
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
asked Nov 26 '18 at 3:12
user3170725user3170725
377
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
asked Nov 26 '18 at 3:12
user3170725user3170725
377
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
edited Nov 26 '18 at 3:44
eyllanesc
77.4k103156
77.4k103156
asked Nov 26 '18 at 3:12
user3170725user3170725
377
asked Nov 26 '18 at 3:12
user3170725user3170725
377
asked Nov 26 '18 at 3:12
user3170725user3170725
377
377
add a comment |
add a comment |
0
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53474334%2ftrying-to-scrap-javascript-with-pyqt4%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
0
active
oldest
votes
0
active
oldest
votes
active
oldest
votes
active
oldest
votes
draft saved
draft discarded
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53474334%2ftrying-to-scrap-javascript-with-pyqt4%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
