Automate the Boring Stuff with Go, Ch. 12: CSV and JSON

// read a CSV file with the standard library, then write one
package main

import (
	"encoding/csv"
	"fmt"
	"os"
	"strings"
)

func main() {
	data := "name,city,age\nAda,London,36\nLinus,Helsinki,54\n\"Grace, Rear Admiral\",Arlington,85\n"

	// csv.Reader handles quotes and commas inside quotes for you
	r := csv.NewReader(strings.NewReader(data))
	rows, err := r.ReadAll()
	if err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
	header, body := rows[0], rows[1:]
	fmt.Println("columns:", header)
	for _, row := range body {
		fmt.Printf("%s lives in %s\n", row[0], row[1])
	}

	// csv.Writer quotes whatever needs quoting
	w := csv.NewWriter(os.Stdout)
	w.Write([]string{"name", "note"})
	w.Write([]string{"Ada", "wrote the first program, in 1843"})
	w.Flush()
	if err := w.Error(); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// a struct in, JSON out, and back again
package main

import (
	"encoding/json"
	"fmt"
	"os"
)

// The tags say what each field is called in JSON. Without a tag, the field name is used
// as written, capital letter and all. Fields that start with a lower-case letter are
// private and never make it into JSON.
type Book struct {
	Title    string   `json:"title"`
	Authors  []string `json:"authors"`
	Year     int      `json:"year"`
	Pages    int      `json:"pages,omitempty"` // left out of the output when zero
	internal string   // unexported: ignored by encoding/json
}

func main() {
	b := Book{Title: "The Go Programming Language", Authors: []string{"Donovan", "Kernighan"}, Year: 2015}

	out, err := json.MarshalIndent(b, "", "  ")
	if err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
	fmt.Println(string(out))

	in := `{"title":"Automate the Boring Stuff with Go","authors":["Toul Cranmer"],"year":2026,"extra":"ignored"}`
	var got Book
	if err := json.Unmarshal([]byte(in), &got); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
	fmt.Printf("%+v\n", got)
}

// JSON whose shape you do not know in advance: decode into any and look around
package main

import (
	"encoding/json"
	"fmt"
	"os"
)

func main() {
	raw := `{"name":"xkcd","comics":[{"num":1,"title":"Barrel"},{"num":2,"title":"Petit Trees"}],"active":true}`

	var v any
	if err := json.Unmarshal([]byte(raw), &v); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}

	// objects become map[string]any, arrays []any, numbers float64, and you assert as you go
	obj := v.(map[string]any)
	fmt.Println("name:", obj["name"])
	for _, c := range obj["comics"].([]any) {
		comic := c.(map[string]any)
		fmt.Printf("#%.0f %s\n", comic["num"], comic["title"])
	}
	if active, ok := obj["active"].(bool); ok && active {
		fmt.Println("still publishing")
	}
}

// csv-remove-header strips the first row from every CSV file in a folder and writes the
// result to a headerRemoved/ sub-folder, leaving the originals alone. It is the job you
// get when a tool insists on a header row and the file you were given has one you do not
// want merged into the data.
//
//	go run ./ch12/projects/csv-remove-header ./ch12/projects/csv-remove-header/sample
package main

import (
	"encoding/csv"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// StripHeader reads src as CSV and writes every row but the first to dst. It returns the
// number of rows written.
func StripHeader(src, dst string) (int, error) {
	in, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	defer in.Close()
	rows, err := csv.NewReader(in).ReadAll()
	if err != nil {
		return 0, fmt.Errorf("parse %s: %w", src, err)
	}
	if len(rows) == 0 {
		return 0, nil
	}
	out, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	w := csv.NewWriter(out)
	if err := w.WriteAll(rows[1:]); err != nil {
		out.Close()
		return 0, fmt.Errorf("write %s: %w", dst, err)
	}
	return len(rows) - 1, out.Close()
}

// ProcessFolder strips every *.csv directly inside folder into folder/headerRemoved.
func ProcessFolder(folder string) (map[string]int, error) {
	outDir := filepath.Join(folder, "headerRemoved")
	if err := os.MkdirAll(outDir, 0o755); err != nil {
		return nil, err
	}
	matches, err := filepath.Glob(filepath.Join(folder, "*.csv"))
	if err != nil {
		return nil, err
	}
	done := map[string]int{}
	for _, src := range matches {
		name := filepath.Base(src)
		if !strings.HasSuffix(strings.ToLower(name), ".csv") {
			continue
		}
		n, err := StripHeader(src, filepath.Join(outDir, name))
		if err != nil {
			return done, err
		}
		done[name] = n
	}
	return done, nil
}

func run(args []string) error {
	if len(args) != 1 {
		return errors.New("usage: csv-remove-header <folder>")
	}
	done, err := ProcessFolder(args[0])
	if err != nil {
		return err
	}
	for name, n := range done {
		fmt.Printf("%s: %d data row(s) kept\n", name, n)
	}
	fmt.Printf("wrote %d file(s) to %s\n", len(done), filepath.Join(args[0], "headerRemoved"))
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// csv-to-json turns an expenses CSV into typed JSON: each row becomes a struct, the
// numbers become numbers, and a summary by category comes along for free.
//
//	go run ./ch12/projects/csv-to-json ./ch12/projects/csv-to-json/expenses.csv
package main

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// Expense is one row. The tags choose the JSON names.
type Expense struct {
	Date     string  `json:"date"`
	Category string  `json:"category"`
	Item     string  `json:"item"`
	Amount   float64 `json:"amount"`
}

// Report is what we write out.
type Report struct {
	Expenses   []Expense          `json:"expenses"`
	ByCategory map[string]float64 `json:"by_category"`
	Total      float64            `json:"total"`
}

// ReadExpenses parses CSV with the header date,category,item,amount in any column order.
func ReadExpenses(r io.Reader) ([]Expense, error) {
	rows, err := csv.NewReader(r).ReadAll()
	if err != nil {
		return nil, err
	}
	if len(rows) == 0 {
		return nil, errors.New("empty file")
	}
	col := map[string]int{}
	for i, name := range rows[0] {
		col[strings.ToLower(strings.TrimSpace(name))] = i
	}
	for _, need := range []string{"date", "category", "item", "amount"} {
		if _, ok := col[need]; !ok {
			return nil, fmt.Errorf("missing column %q", need)
		}
	}
	var out []Expense
	for n, row := range rows[1:] {
		amount, err := strconv.ParseFloat(strings.TrimSpace(row[col["amount"]]), 64)
		if err != nil {
			return nil, fmt.Errorf("row %d: amount %q: %w", n+2, row[col["amount"]], err)
		}
		out = append(out, Expense{
			Date:     row[col["date"]],
			Category: row[col["category"]],
			Item:     row[col["item"]],
			Amount:   amount,
		})
	}
	return out, nil
}

// Summarize builds the report from the rows.
func Summarize(expenses []Expense) Report {
	rep := Report{Expenses: expenses, ByCategory: map[string]float64{}}
	for _, e := range expenses {
		rep.ByCategory[e.Category] += e.Amount
		rep.Total += e.Amount
	}
	return rep
}

func run(args []string) error {
	if len(args) != 1 {
		return errors.New("usage: csv-to-json <expenses.csv>")
	}
	f, err := os.Open(args[0])
	if err != nil {
		return err
	}
	defer f.Close()
	expenses, err := ReadExpenses(f)
	if err != nil {
		return fmt.Errorf("%s: %w", args[0], err)
	}
	out := strings.TrimSuffix(args[0], ".csv") + ".json"
	data, err := json.MarshalIndent(Summarize(expenses), "", "  ")
	if err != nil {
		return err
	}
	if err := os.WriteFile(out, data, 0o644); err != nil {
		return err
	}
	fmt.Printf("%d expense(s), total %.2f, wrote %s\n", len(expenses), Summarize(expenses).Total, out)
	return nil
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

// json-get pulls one value out of a JSON document by path, the way you would with jq,
// and pretty-prints the whole document when you give it no path. It reads a file or
// standard input, so it chains with the other tools in this book.
//
//	go run ./ch12/projects/json-get ./ch9/projects/census-by-county/census_summary.json
//	curl -s https://xkcd.com/info.0.json | go run ./ch12/projects/json-get -path title
//	go run ./ch12/projects/json-get -path comics.1.title ./ch12/projects/json-get/sample.json
package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// Get walks v along a dotted path. Object keys are names; array elements are indexes.
func Get(v any, path string) (any, error) {
	if path == "" {
		return v, nil
	}
	for _, part := range strings.Split(path, ".") {
		switch node := v.(type) {
		case map[string]any:
			next, ok := node[part]
			if !ok {
				return nil, fmt.Errorf("no key %q", part)
			}
			v = next
		case []any:
			i, err := strconv.Atoi(part)
			if err != nil || i < 0 || i >= len(node) {
				return nil, fmt.Errorf("no element %q in an array of %d", part, len(node))
			}
			v = node[i]
		default:
			return nil, fmt.Errorf("cannot descend into %T with %q", v, part)
		}
	}
	return v, nil
}

// Render prints strings bare and everything else as indented JSON, which is what you
// want when the result goes straight into another command.
func Render(v any) (string, error) {
	if s, ok := v.(string); ok {
		return s, nil
	}
	out, err := json.MarshalIndent(v, "", "  ")
	return string(out), err
}

func run(args []string, stdin io.Reader) error {
	flags := flag.NewFlagSet("json-get", flag.ContinueOnError)
	path := flags.String("path", "", "dotted path, e.g. comics.1.title (empty = whole document)")
	if err := flags.Parse(args); err != nil {
		return err
	}
	var in io.Reader = stdin
	switch flags.NArg() {
	case 0:
	case 1:
		f, err := os.Open(flags.Arg(0))
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	default:
		return errors.New("usage: json-get [-path a.b.0] [file.json]")
	}
	var v any
	if err := json.NewDecoder(in).Decode(&v); err != nil {
		return fmt.Errorf("parse JSON: %w", err)
	}
	got, err := Get(v, *path)
	if err != nil {
		return err
	}
	s, err := Render(got)
	if err != nil {
		return err
	}
	fmt.Println(s)
	return nil
}

func main() {
	if err := run(os.Args[1:], os.Stdin); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}