Automate the Boring Stuff with Go, Ch. 11: Web Requests

June 16, 2023 · 5 min read · #golang, #automate-the-boring-stuff-with-go

// written in 2023: tools, versions and prices may have changed since.

Part of Automate the Boring Stuff with Go, a beginner-friendly guide to programming in Go.

Request

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

func main() {

	pwdDir, _ := os.Getwd()
	fLog, err := os.OpenFile(pwdDir+"/log.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
	check(err, fLog)

	url := "http://www.gutenberg.org/cache/epub/1112/pg1112.txt"

	resp, err := http.Get(url)
	check(err, fLog)

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	check(err, fLog)

	fmt.Println(string(body))
}

// err check to log
func check(err error, fLog *os.File) {
	if err != nil {
		log.Fatalln(err)
	}
	log.SetOutput(io.MultiWriter(fLog, os.Stdout))
}

Search Term

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/jackdanger/collectlinks"
)

func main() {

	pwdDir, _ := os.Getwd()
	fLog, err := os.OpenFile(pwdDir+"/log.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
	check(err, fLog)

	resp, err := http.Get("https://automatetheboringstuffwithgo.com")
	check(err, fLog)
	defer resp.Body.Close()

	links := collectlinks.All(resp.Body)

	for _, link := range links {
		fmt.Println(link)
	}
}

// err check to log
func check(err error, fLog *os.File) {
	if err != nil {
		log.Fatalln(err)
	}
	log.SetOutput(io.MultiWriter(fLog, os.Stdout))
}

Request Save

package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

func main() {
	pwdDir, _ := os.Getwd()
	fLog, err := os.OpenFile(pwdDir+"/log.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
	check(err, fLog)

	fSave, err := os.OpenFile(pwdDir+"/save.txt", os.O_CREATE|os.O_WRONLY, 0600)
	check(err, fLog)

	url := "http://www.gutenberg.org/cache/epub/1112/pg1112.txt"

	resp, err := http.Get(url)
	check(err, fLog)
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	check(err, fLog)

	fSave.Write(body)
}

// err check to log
func check(err error, fLog *os.File) {
	if err != nil {
		log.Fatalln(err)
	}
	log.SetOutput(io.MultiWriter(fLog, os.Stdout))
}

Download XKCD

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"

	"strings"

	"strconv"

	"sort"

	"github.com/opesun/goquery"
)

func main() {
	pwdDir, _ := os.Getwd()
	fLog, err := os.OpenFile(pwdDir+"/log.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640)
	check(err, fLog)

	url := `http://xkcd.com`

	err = os.Mkdir(pwdDir+"/xkcd", 0775)
	check(err, fLog)

	resp, err := http.Get(url)
	log.Printf("URL:	%v", url)
	check(err, fLog)
	defer resp.Body.Close()

	x, err := goquery.Parse(resp.Body)
	check(err, fLog)

	var urlInt []int
	regLastLink, _ := regexp.Compile(`\/[0-9]{1,}\/`)
	for _, i := range x.Find("a").Attrs("href") {
		if regLastLink.MatchString(i) {
			a, err := strconv.Atoi(strings.Trim(i, "/"))
			check(err, fLog)
			urlInt = append(urlInt, a)
		}
	}
	sort.Ints(urlInt)
	fmt.Println(urlInt[len(urlInt)-1])

	for k := urlInt[len(urlInt)-1] + 1; k >= 1; k-- {
		resp, err := http.Get(url + "/" + strconv.Itoa(k))
		log.Printf("URL: 	%v", url+"/"+strconv.Itoa(k))
		check(err, fLog)
		defer resp.Body.Close()

		x, err := goquery.Parse(resp.Body)
		check(err, fLog)

		regStr, _ := regexp.Compile(`comics`)
		for _, i := range x.Find("img").Attrs("src") {
			if regStr.MatchString(i) {

				nameFile := strings.Split(i, "/comics/")

				name := pwdDir + "/xkcd/" + nameFile[1]
				fSave, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY, 0640)
				check(err, fLog)

				respImg, err := http.Get("http:" + i)
				log.Printf("http	%v", "http:"+i)
				check(err, fLog)

				defer respImg.Body.Close()

				bodyImg, err := ioutil.ReadAll(respImg.Body)
				check(err, fLog)

				log.Printf("name:	%v", name)
				fSave.Write(bodyImg)
			}
		}
	}

	log.SetOutput(io.MultiWriter(fLog, os.Stdout))
}

// err check to log
func check(err error, fLog *os.File) {
	if err != nil {
		log.Println(err)
	}
	log.SetOutput(io.MultiWriter(fLog, os.Stdout))
}

Selenium

package main

import (
	"encoding/csv"
	"github.com/tebeka/selenium"
	"github.com/tebeka/selenium/chrome"
	"log"
	"os"
)

// define a custom data type for the scraped data
type Product struct {
	name, price string
}

func main() {
	// where to store the scraped data
	var products []Product

	// initialize a Chrome browser instance on port 4444
	service, err := selenium.NewChromeDriverService("./chromedriver/chromedriver", 4444)
	if err != nil {
		log.Fatal("Error:", err)
	}

	defer service.Stop()

	// configure the browser options
	caps := selenium.Capabilities{}
	caps.AddChrome(chrome.Capabilities{Args: []string{
		"--headless", // comment out this line for testing
	}})

	// create a new remote client with the specified options
	driver, err := selenium.NewRemote(caps, "")
	if err != nil {
		log.Fatal("Error:", err)
	}

	// maximize the current window to avoid responsive rendering
	err = driver.MaximizeWindow("")
	if err != nil {
		log.Fatal("Error:", err)
	}

	// visit the target page
	err = driver.Get("https://scrapingclub.com/exercise/list_infinite_scroll/")
	if err != nil {
		log.Fatal("Error:", err)
	}

	// select the product elements
	productElements, err := driver.FindElements(selenium.ByCSSSelector, ".post")
	if err != nil {
		log.Fatal("Error:", err)
	}

	// iterate over the product elements
	// and extract data from them
	for _, productElement := range productElements {
		// select the name and price nodes
		nameElement, err := productElement.FindElement(selenium.ByCSSSelector, "h4")
		priceElement, err := productElement.FindElement(selenium.ByCSSSelector, "h5")
		// extract the data of interest
		name, err := nameElement.Text()
		price, err := priceElement.Text()
		if err != nil {
			log.Fatal("Error:", err)
		}

		// add the scraped data to the list
		product := Product{}
		product.name = name
		product.price = price
		products = append(products, product)
	}

	// export the scraped data to CSV
	file, err := os.Create("products.csv")
	if err != nil {
		log.Fatal("Error:", err)
	}

	defer file.Close()

	// initialize a file writer
	writer := csv.NewWriter(file)

	// define the CSV headers
	headers := []string{
		"name",
		"price",
	}

	// write the column headers
	writer.Write(headers)

	// adding each product to the CSV output file
	for _, product := range products {

		// converting a Product to an array of strings
		record := []string{
			product.name,
			product.price,
		}

		// writing a new CSV record
		writer.Write(record)
	}

	defer writer.Flush()
}

Google Search

package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/http"
	"os"
)

func getData() {
	// change the query t be a var
	// pass in single search term
	url := fmt.Sprintf("https://www.google.com/search?q=%s&gl=us&hl=en", os.Args[1])
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36")
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()

	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	c := 0
	doc.Find("div.g").Each(func(i int, result *goquery.Selection) {
		title := result.Find("h3").First().Text()
		link, _ := result.Find("a").First().Attr("href")
		snippet := result.Find(".VwiC3b").First().Text()

		fmt.Printf("Title: %s\n", title)
		fmt.Printf("Link: %s\n", link)
		fmt.Printf("Snippet: %s\n", snippet)
		fmt.Printf("Position: %d\n", c+1)
		fmt.Println()

		c++
	})
}

func main() {
	getData()
}

cd ../blog