gcd_downloader/main.go

package main

import (
	"archive/zip"
	"context"
	"flag"
	"io"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/andybalholm/brotli"
	"github.com/kr/pretty"
	"golang.org/x/net/publicsuffix"
)

type client struct {
	*http.Client
	referrer string
}

func Fatal(a ...any) {
	log.Print(a...)
	os.Exit(125)
}

func getForm(selector string, resp *http.Response) (action string, form url.Values) {
	var body io.Reader = resp.Body
	if resp.Header.Get("Content-Encoding") == "br" {
		body = brotli.NewReader(resp.Body)
	}
	doc, err := goquery.NewDocumentFromReader(body)
	if err != nil {
		Fatal(err)
	}
	form = url.Values{}
	if sel := doc.Find(selector); sel != nil && len(sel.Nodes) == 1 {
		action = sel.AttrOr("action", "")
		sel.Find("input").Each(func(i int, s *goquery.Selection) {
			form.Add(s.AttrOr("name", s.AttrOr("id", "")), s.AttrOr("value", ""))
		})
	} else {
		Fatal(doc.Html())
	}
	return action, form
}

func (cl *client) do(ctx context.Context, method string, url string, body io.Reader) (*http.Response, error) {
	req, _ := http.NewRequestWithContext(ctx, method, url, body)
	req.Header.Set("Referer", cl.referrer)
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
	resp, err := cl.Do(req)
	cl.referrer = url
	return resp, err
}

func (cl *client) Post(ctx context.Context, url, contentType string, body io.Reader) (resp *http.Response, err error) {
	req, _ := http.NewRequestWithContext(ctx, "POST", url, body)
	req.Header.Set("Referer", cl.referrer)
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
	req.Header.Set("Content-Type", contentType)
	resp, err = cl.Do(req)
	cl.referrer = url
	return resp, err
}

func Redirections(resp *http.Response) (history []*http.Request) {
	for resp != nil {
		req := resp.Request
		history = append(history, req)
		resp = req.Response
	}
	for l, r := 0, len(history)-1; l < r; l, r = l+1, r-1 {
		history[l], history[r] = history[r], history[l]
	}
	return history
}

func printResp(resp *http.Response) {
	for _, req := range Redirections(resp) {
	log.Println("URL:", req.URL)
	pretty.Println("Sent Headers:", req.Header)
}

	log.Println("Status:", resp.Status)
	pretty.Println(resp.Header)

}

var (
	sqlite_filename string
	sqlite_br_filename string
	destination *string = flag.String("destination", "/perm/www/items", "destination directory for sqlite files")
	workDir *string = flag.String("work", "/perm/workDir", "working directory for sqlite files")
)

func downlod() {
	u, _ := url.Parse("https://www.comics.org/accounts/login/?next=/download/")
	cl := client{&http.Client{}, ""}
	cl.Jar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	resp, err := cl.do(ctx, "GET", u.String(), nil)
	if err != nil {
		Fatal(err)
	}
	// printResp(resp)

	if resp.StatusCode != http.StatusOK {
		Fatal(resp.Status)
	}
	var (
		action string
		form url.Values
	)

	action, form = getForm(`div.body_content > form[action*="/accounts/login/"]`, resp)
	resp.Body.Close()
	form.Set("username", os.Getenv("GCD_USERNAME"))
	form.Set("password", os.Getenv("GCD_PASSWORD"))
	form.Del("")

	ctx, cancel = context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	resp, err = cl.Post(ctx, u.ResolveReference(&url.URL{Path: action}).String(), "application/x-www-form-urlencoded", strings.NewReader(form.Encode()))
	if err != nil {
		Fatal(err)
	}
	// printResp(resp)

	if resp.StatusCode != http.StatusOK {
		Fatal(resp.Status)
	}
	action, form = getForm(`div.body_content > form.download`, resp)
	resp.Body.Close()
	form.Del("mysqldump")
	form.Del("name-value")
	form.Del("")
	form.Set("accept_license", "on")
	form.Set("purpose", "non-commercial")

	ctx, cancel = context.WithTimeout(context.Background(), time.Minute*15)
	defer cancel()
	resp, err = cl.Post(ctx, u.ResolveReference(&url.URL{Path: action}).String(), "application/x-www-form-urlencoded", strings.NewReader(form.Encode()))
	if err != nil {
		Fatal(err)
	}
	// printResp(resp)

	if resp.StatusCode != http.StatusOK {
		Fatal(resp.Status)
	}
	defer resp.Body.Close()
	file, err := os.Create(filepath.Join(*workDir, "temp.zip"))
	if err != nil {
		Fatal(err)
	}
	defer file.Close()

	log.Println("Downloading sqlite file now")
	_, err = io.Copy(file, resp.Body)
	if err != nil {
		Fatal(err)
	}
	log.Println("Download Complete")

	size, err := file.Seek(0, io.SeekEnd)
	if err != nil {
		Fatal(err)
	}
	_, err = file.Seek(0, io.SeekStart)
	if err != nil {
		Fatal(err)
	}

	zipFile, err := zip.NewReader(file, size)
	if err != nil {
		Fatal(err)
	}

	sqlite_filename = zipFile.File[0].Name
	sqlite_br_filename = zipFile.File[0].Name+".br"

	sqlite, err :=zipFile.File[0].Open()
	if err != nil {
		Fatal(err)
	}
	defer sqlite.Close()

	sqlite_file, err := os.Create(filepath.Join(*workDir, sqlite_filename))
	if err != nil {
		Fatal(err)
	}
	defer sqlite_file.Close()

	sqlite_br_file, err := os.Create(filepath.Join(*workDir, sqlite_br_filename))
	if err != nil {
		Fatal(err)
	}
	defer sqlite_br_file.Close()

	sqlite_reader := io.TeeReader(sqlite, sqlite_file)

	br := brotli.NewWriterLevel(sqlite_br_file, 5)

	log.Printf("starting compression of %s to %s\n", sqlite_filename, sqlite_br_filename)

	_, err = io.Copy(br, sqlite_reader)
	if err != nil {
		Fatal(err)
	}

	err = os.Rename(filepath.Join(*workDir, sqlite_filename), filepath.Join(*destination, sqlite_filename))
	if err != nil {
		Fatal(err)
	}
	err = os.Rename(filepath.Join(*workDir, sqlite_br_filename), filepath.Join(*destination, sqlite_br_filename))
	if err != nil {
		Fatal(err)
	}

	err = os.Remove(filepath.Join(*workDir, "temp.zip"))
	if err != nil {
		Fatal(err)
	}

	log.Println("complete")
}

func createDirs() {
	err := os.MkdirAll(*workDir, 0o777)
	if err != nil {
		Fatal(err)
	}
	err = os.MkdirAll(*destination, 0o777)
	if err != nil {
		Fatal(err)
	}
}

func shouldUpdate() bool {
	dir, err := os.Open(*destination)
	if err != nil {
		Fatal(err)
	}
	names, err := dir.Readdirnames(0)
	if err != nil {
		Fatal(err)
	}
	var names2 []string
	for _, name := range names {
		if len(name) != 13 {
			continue
		}
		names2 = append(names2, name)
	}
	sort.Strings(names2)
	if len(names2) < 1 {
		return true
	}
	latest := names2[len(names2)-1]
	lastUpdate, _ := time.Parse("2006-01-02.db", latest)
	timeTillUpdate := lastUpdate.Sub(time.Now().UTC().AddDate(0,0,-7))
	if lastUpdate.Before(time.Now().UTC().AddDate(0,0,-7)) {
		return true
	}
	log.Printf("can't update for another %v", timeTillUpdate)
	return false
}

func main() {
	flag.Parse()
	if os.Getenv("GCD_USERNAME") == "" || os.Getenv("GCD_PASSWORD") == "" {
		log.Println("Username and password not provided")
		os.Exit(0)
	}
	for {
		createDirs()
		if shouldUpdate() {
			downlod()
			log.Println("update complete, waiting 1h to check again")
		} else {
			log.Println("It's too early, waiting 1h to check again")
		}
		time.Sleep(time.Hour)
	}
}