Scrapemate is a web crawling and scraping framework written in Golang. It is designed to be simple and easy to use, yet powerful enough to handle complex scraping tasks.
- Low level API & Easy High Level API
- Customizable retry and error handling
- Javascript Rendering with ability to control the browser
- Screenshots support (when JS rendering is enabled)
- Capability to write your own result exporter
- Capability to write results in multiple sinks
- Default CSV writer
- Caching (File/LevelDB/Custom)
- Custom job providers (memory provider included)
- Headless and Headful support when using JS rendering
- Automatic cookie and session handling
- Rotating HTTP/HTTPS/SOCKS5 proxy support
Scrapemate uses Playwright for JavaScript rendering. It requires the Playwright browsers to be installed.
# Install playwright browsers
go run github.com/playwright-community/playwright-go/cmd/playwright install --with-deps chromiumBuild and run without any special tags:
go build ./...The books-to-scrape-simple example demonstrates JavaScript rendering with Playwright:
# Run with Playwright (default)
# First install browsers: go run github.com/playwright-community/playwright-go/cmd/playwright install --with-deps chromium
go run . -jsgo get github.com/gosom/scrapemate
package main
import (
"context"
"encoding/csv"
"fmt"
"net/http"
"os"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/gosom/scrapemate"
"github.com/gosom/scrapemate/adapters/writers/csvwriter"
"github.com/gosom/scrapemate/scrapemateapp"
)
func main() {
csvWriter := csvwriter.NewCsvWriter(csv.NewWriter(os.Stdout))
cfg, err := scrapemateapp.NewConfig(
[]scrapemate.ResultWriter{csvWriter},
)
if err != nil {
panic(err)
}
app, err := scrapemateapp.NewScrapeMateApp(cfg)
if err != nil {
panic(err)
}
seedJobs := []scrapemate.IJob{
&SimpleCountryJob{
Job: scrapemate.Job{
ID: "identity",
Method: http.MethodGet,
URL: "https://www.scrapethissite.com/pages/simple/",
Headers: map[string]string{
"User-Agent": scrapemate.DefaultUserAgent,
},
Timeout: 10 * time.Second,
MaxRetries: 3,
},
},
}
err = app.Start(context.Background(), seedJobs...)
if err != nil && err != scrapemate.ErrorExitSignal {
panic(err)
}
}
type SimpleCountryJob struct {
scrapemate.Job
}
func (j *SimpleCountryJob) Process(ctx context.Context, resp *scrapemate.Response) (any, []scrapemate.IJob, error) {
doc, ok := resp.Document.(*goquery.Document)
if !ok {
return nil, nil, fmt.Errorf("failed to cast response document to goquery document")
}
var countries []Country
doc.Find("div.col-md-4.country").Each(func(i int, s *goquery.Selection) {
var country Country
country.Name = strings.TrimSpace(s.Find("h3.country-name").Text())
country.Capital = strings.TrimSpace(s.Find("div.country-info span.country-capital").Text())
country.Population = strings.TrimSpace(s.Find("div.country-info span.country-population").Text())
country.Area = strings.TrimSpace(s.Find("div.country-info span.country-area").Text())
countries = append(countries, country)
})
return countries, nil, nil
}
type Country struct {
Name string
Capital string
Population string
Area string
}
func (c Country) CsvHeaders() []string {
return []string{"Name", "Capital", "Population", "Area"}
}
func (c Country) CsvRow() []string {
return []string{c.Name, c.Capital, c.Population, c.Area}
}go mod tidy
go run main.go 1>countries.csv
(hit CTRL-C to exit)
Version 1.0.0 introduces a BrowserPage interface abstraction for browser automation. This is a breaking change for users who use JavaScript rendering with BrowserActions.
// Before (v0.9.x)
func (j *MyJob) BrowserActions(ctx context.Context, page playwright.Page) scrapemate.Response {
page.Goto("https://example.com", playwright.PageGotoOptions{
WaitUntil: playwright.WaitUntilStateNetworkidle,
})
html, _ := page.Content()
return scrapemate.Response{Body: []byte(html)}
}
// After (v1.0.0)
func (j *MyJob) BrowserActions(ctx context.Context, page scrapemate.BrowserPage) scrapemate.Response {
resp, err := page.Goto("https://example.com", scrapemate.WaitUntilNetworkIdle)
if err != nil {
return scrapemate.Response{Error: err}
}
return scrapemate.Response{
Body: resp.Body,
StatusCode: resp.StatusCode,
}
}If you need browser-specific features, use Unwrap():
// For Playwright
pwPage := page.Unwrap().(playwright.Page)See CHANGELOG.md for the full list of changes.
You can find more documentation here
For the High Level API see this example.
Read also how to use high level api
For the Low Level API see books.toscrape.com
Additionally, for low level API you can read the blogpost
See an example of how you can use scrapemate go scrape Google Maps: https://github.com/gosom/google-maps-scraper
Contributions are welcome.
Scrapemate is licensed under the MIT License. See LICENCE file