package main import ( "archive/zip" "context" "flag" "io" "log" "net/http" "net/http/cookiejar" "net/url" "os" "path/filepath" "sort" "strings" "time" "github.com/PuerkitoBio/goquery" "github.com/andybalholm/brotli" "github.com/kr/pretty" "golang.org/x/net/publicsuffix" ) type client struct { *http.Client referrer string } func Fatal(a ...any) { log.Print(a...) os.Exit(125) } func getForm(selector string, resp *http.Response) (action string, form url.Values) { var body io.Reader = resp.Body if resp.Header.Get("Content-Encoding") == "br" { body = brotli.NewReader(resp.Body) } doc, err := goquery.NewDocumentFromReader(body) if err != nil { Fatal(err) } form = url.Values{} if sel := doc.Find(selector); sel != nil && len(sel.Nodes) == 1 { action = sel.AttrOr("action", "") sel.Find("input").Each(func(i int, s *goquery.Selection) { form.Add(s.AttrOr("name", s.AttrOr("id", "")), s.AttrOr("value", "")) }) } else { Fatal(doc.Html()) } return action, form } func (cl *client) do(ctx context.Context, method string, url string, body io.Reader) (*http.Response, error) { req, _ := http.NewRequestWithContext(ctx, method, url, body) req.Header.Set("Referer", cl.referrer) req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req.Header.Set("Accept-Encoding", "gzip, deflate, br") resp, err := cl.Do(req) cl.referrer = url return resp, err } func (cl *client) Post(ctx context.Context, url, contentType string, body io.Reader) (resp *http.Response, err error) { req, _ := http.NewRequestWithContext(ctx, "POST", url, body) req.Header.Set("Referer", cl.referrer) req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req.Header.Set("Accept-Encoding", "gzip, deflate, br") req.Header.Set("Content-Type", contentType) resp, err = cl.Do(req) cl.referrer = url return resp, err } func Redirections(resp *http.Response) (history []*http.Request) { for resp != nil { req := resp.Request history = append(history, req) resp = req.Response } for l, r := 0, len(history)-1; l < r; l, r = l+1, r-1 { history[l], history[r] = history[r], history[l] } return history } func printResp(resp *http.Response) { for _, req := range Redirections(resp) { log.Println("URL:", req.URL) pretty.Println("Sent Headers:", req.Header) } log.Println("Status:", resp.Status) pretty.Println(resp.Header) } var ( sqlite_filename string sqlite_br_filename string destination *string = flag.String("destination", "/perm/www/items", "destination directory for sqlite files") workDir *string = flag.String("work", "/perm/workDir", "working directory for sqlite files") ) func downlod() { u, _ := url.Parse("https://www.comics.org/accounts/login/?next=/download/") cl := client{&http.Client{}, ""} cl.Jar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() resp, err := cl.do(ctx, "GET", u.String(), nil) if err != nil { Fatal(err) } // printResp(resp) if resp.StatusCode != http.StatusOK { Fatal(resp.Status) } var ( action string form url.Values ) action, form = getForm(`div.body_content > form[action*="/accounts/login/"]`, resp) resp.Body.Close() form.Set("username", os.Getenv("GCD_USERNAME")) form.Set("password", os.Getenv("GCD_PASSWORD")) form.Del("") ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() resp, err = cl.Post(ctx, u.ResolveReference(&url.URL{Path: action}).String(), "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) if err != nil { Fatal(err) } // printResp(resp) if resp.StatusCode != http.StatusOK { Fatal(resp.Status) } action, form = getForm(`div.body_content > form.download`, resp) resp.Body.Close() form.Del("mysqldump") form.Del("name-value") form.Del("") form.Set("accept_license", "on") form.Set("purpose", "non-commercial") ctx, cancel = context.WithTimeout(context.Background(), time.Minute*15) defer cancel() resp, err = cl.Post(ctx, u.ResolveReference(&url.URL{Path: action}).String(), "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) if err != nil { Fatal(err) } // printResp(resp) if resp.StatusCode != http.StatusOK { Fatal(resp.Status) } defer resp.Body.Close() file, err := os.Create(filepath.Join(*workDir, "temp.zip")) if err != nil { Fatal(err) } defer file.Close() log.Println("Downloading sqlite file now") _, err = io.Copy(file, resp.Body) if err != nil { Fatal(err) } log.Println("Download Complete") size, err := file.Seek(0, io.SeekEnd) if err != nil { Fatal(err) } _, err = file.Seek(0, io.SeekStart) if err != nil { Fatal(err) } zipFile, err := zip.NewReader(file, size) if err != nil { Fatal(err) } sqlite_filename = zipFile.File[0].Name sqlite_br_filename = zipFile.File[0].Name+".br" sqlite, err :=zipFile.File[0].Open() if err != nil { Fatal(err) } defer sqlite.Close() sqlite_file, err := os.Create(filepath.Join(*workDir, sqlite_filename)) if err != nil { Fatal(err) } defer sqlite_file.Close() sqlite_br_file, err := os.Create(filepath.Join(*workDir, sqlite_br_filename)) if err != nil { Fatal(err) } defer sqlite_br_file.Close() sqlite_reader := io.TeeReader(sqlite, sqlite_file) br := brotli.NewWriterLevel(sqlite_br_file, 5) log.Printf("starting compression of %s to %s\n", sqlite_filename, sqlite_br_filename) _, err = io.Copy(br, sqlite_reader) if err != nil { Fatal(err) } err = os.Rename(filepath.Join(*workDir, sqlite_filename), filepath.Join(*destination, sqlite_filename)) if err != nil { Fatal(err) } err = os.Rename(filepath.Join(*workDir, sqlite_br_filename), filepath.Join(*destination, sqlite_br_filename)) if err != nil { Fatal(err) } err = os.Remove(filepath.Join(*workDir, "temp.zip")) if err != nil { Fatal(err) } log.Println("complete") } func createDirs() { err := os.MkdirAll(*workDir, 0o777) if err != nil { Fatal(err) } err = os.MkdirAll(*destination, 0o777) if err != nil { Fatal(err) } } func shouldUpdate() bool { dir, err := os.Open(*destination) if err != nil { Fatal(err) } names, err := dir.Readdirnames(0) if err != nil { Fatal(err) } var names2 []string for _, name := range names { if len(name) != 13 { continue } names2 = append(names2, name) } sort.Strings(names2) if len(names2) < 1 { return true } latest := names2[len(names2)-1] lastUpdate, _ := time.Parse("2006-01-02.db", latest) timeTillUpdate := lastUpdate.Sub(time.Now().UTC().AddDate(0,0,-7)) if lastUpdate.Before(time.Now().UTC().AddDate(0,0,-7)) { return true } log.Printf("can't update for another %v", timeTillUpdate) return false } func main() { flag.Parse() if os.Getenv("GCD_USERNAME") == "" || os.Getenv("GCD_PASSWORD") == "" { log.Println("Username and password not provided") os.Exit(0) } for { createDirs() if shouldUpdate() { downlod() log.Println("update complete, waiting 1h to check again") } else { log.Println("It's too early, waiting 1h to check again") } time.Sleep(time.Hour) } }