Skip to content

Commit 22a5435

Browse files
xxg1413xxg1413
committed
add Chapter13
1 parent 7c8003e commit 22a5435

8 files changed

Lines changed: 139 additions & 0 deletions

File tree

‎Web Scraping with Python/Chapter13/combined-test.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
from selenium import webdriver
2+
from selenium.webdriver.remote.webelement import WebElement
3+
from selenium.webdriver import ActionChains
4+
import unittest
5+
6+
class TestAddition(unittest.TestCase):
7+
driver = None
8+
def setUp(self):
9+
global driver
10+
driver = webdriver.PhantomJS(executable_path='/Users/xxg/phantomjs-2.1.1-macosx/bin/phantomjs')
11+
url = 'http://pythonscraping.com/pages/javascript/draggableDemo.html'
12+
driver.get(url)
13+
14+
def tearDown(self):
15+
print("Tearing down the test")
16+
17+
def test_drag(self):
18+
global driver
19+
element = driver.find_element_by_id("draggable")
20+
target = driver.find_element_by_id("div2")
21+
actions = ActionChains(driver)
22+
actions.drag_and_drop(element, target).perform()
23+
24+
self.assertEqual("You are definitely not a bot!", driver.find_element_by_id("message").text)
25+
26+
if __name__ == '__main__':
27+
unittest.main()

‎Web Scraping with Python/Chapter13/drag-and-drop.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from selenium import webdriver
2+
from selenium.webdriver.remote.webelement import WebElement
3+
from selenium.webdriver import ActionChains
4+
5+
driver = webdriver.PhantomJS(executable_path='phantomjs/bin/phantomjs')
6+
driver.get('http://pythonscraping.com/pages/javascript/draggableDemo.html')
7+
8+
print(driver.find_element_by_id("message").text)
9+
10+
element = driver.find_element_by_id("draggable")
11+
target = driver.find_element_by_id("div2")
12+
actions = ActionChains(driver)
13+
actions.drag_and_drop(element, target).perform()
14+
15+
print(driver.find_element_by_id("message").text)

‎Web Scraping with Python/Chapter13/github.png‎

226 KB
Loading

‎Web Scraping with Python/Chapter13/interactive-test.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
from selenium import webdriver
2+
from selenium.webdriver.remote.webelement import WebElement
3+
from selenium.webdriver.common.keys import Keys
4+
from selenium.webdriver import ActionChains
5+
6+
7+
driver = webdriver.PhantomJS(executable_path='phantomjs/bin/phantomjs')
8+
driver.get("http://pythonscraping.com/pages/files/form.html")
9+
10+
firstnameField = driver.find_element_by_name("firstname")
11+
lastnameField = driver.find_element_by_name("lastname")
12+
submitButton = driver.find_element_by_id("submit")
13+
14+
### METHOD 1 ###
15+
firstnameField.send_keys("Ryan")
16+
lastnameField.send_keys("Mitchell")
17+
submitButton.click()
18+
################
19+
20+
### METHOD 2 ###
21+
actions = ActionChains(driver).click(firstnameField).send_keys("Ryan").click(lastnameField).send_keys("Mitchell").send_keys(Keys.RETURN)
22+
actions.perform()
23+
################
24+
25+
print(driver.find_element_by_tag_name("body").text)
26+
27+
driver.close()

‎Web Scraping with Python/Chapter13/pythonscraping.png‎

325 KB
Loading

‎Web Scraping with Python/Chapter13/take-screenshot.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from selenium import webdriver
2+
from selenium.webdriver.remote.webelement import WebElement
3+
from selenium.webdriver import ActionChains
4+
5+
6+
driver = webdriver.PhantomJS(executable_path='/Users/xxg/phantomjs-2.1.1-macosx/bin/phantomjs')
7+
driver.implicitly_wait(5)
8+
driver.get('http://github.com/xxg1413')
9+
driver.get_screenshot_as_file('github.png')

‎Web Scraping with Python/Chapter13/wiki-selenium-test.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
from selenium import webdriver
2+
3+
4+
driver = webdriver.PhantomJS(executable_path='/Users/xxg/phantomjs-2.1.1-macosx/bin/phantomjs')
5+
driver.get("http://en.wikipedia.org/wiki/Monty_Python")
6+
assert "Monty Python" in driver.title
7+
print("Monty Python was not in the title")
8+
driver.close()

‎Web Scraping with Python/Chapter13/wiki-unit-test.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from urllib.request import urlopen
2+
from urllib.parse import unquote
3+
import random
4+
import re
5+
from bs4 import BeautifulSoup
6+
import unittest
7+
8+
class TestWikipedia(unittest.TestCase):
9+
10+
bsObj = None
11+
url = None
12+
13+
14+
def test_PageProperties(self):
15+
global bsObj
16+
global url
17+
18+
url = "http://en.wikipedia.org/wiki/Monty_Python"
19+
#Test the first 100 pages we encounter
20+
for i in range(1, 100):
21+
bsObj = BeautifulSoup(urlopen(url), "html.parser")
22+
titles = self.titleMatchesURL()
23+
self.assertEquals(titles[0], titles[1])
24+
self.assertTrue(self.contentExists())
25+
url = self.getNextLink()
26+
print("Done!")
27+
28+
#测试标题
29+
def titleMatchesURL(self):
30+
global bsObj
31+
global url
32+
pageTitle = bsObj.find("h1").get_text()
33+
urlTitle = url[(url.index("/wiki/")+6):]
34+
urlTitle = urlTitle.replace("_", " ")
35+
urlTitle = unquote(urlTitle)
36+
return [pageTitle.lower(), urlTitle.lower()]
37+
38+
def contentExists(self):
39+
global bsObj
40+
content = bsObj.find("div",{"id":"mw-content-text"})
41+
if content is not None:
42+
return True
43+
return False
44+
45+
def getNextLink(self):
46+
global bsObj
47+
links = bsObj.find("div", {"id":"bodyContent"}).findAll("a", href=re.compile("^(/wiki/)((?!:).)*$"))
48+
link = links[random.randint(0, len(links)-1)].attrs['href']
49+
print("Next link is: "+link)
50+
return "http://en.wikipedia.org"+link
51+
52+
if __name__ == '__main__':
53+
unittest.main()

0 commit comments

Comments
 (0)