lordwelch/gloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor cookie handling

Browse Source 
Switch to jettison for JSON marshaling
Create an HTTP UI
Use goembed for assets
This commit is contained in:
lordwelch 2020-12-21 01:15:16 -08:00
parent bd0c3d2cc6
commit 36e79aa907
19 changed files with 968 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
assets/bootstrap-table.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

10
assets/bootstrap-table.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

7
assets/bootstrap.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

7
assets/bootstrap.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

85
assets/downloads.js Normal file
View File

@ -0,0 +1,85 @@
function updateDownloads(downloads) {
var dls = [];
var i = 0;
var tbody = $("<tbody></tbody>").attr("id", "table")
for (var d of downloads) {
var row = $("<tr></tr>").addClass("download").attr("index", i)
var index = $("<th></th>").attr("scope", "row").text(i)
var filename = $("<td></td>").text(d.subdir + "/" + d.filename).addClass("lastlog")
var url = $("<td></td>").text(d.url)
var status = $("<td></td>").addClass(d.status).css("cursor", "pointer").text(d.status + "\n" + d.error).on("click",{index: i, download: d}, evt => {
if (d.status != "Queue" || d.status != "Complete") {
setDownload([{
index: evt.data("index"),
status: "Queue",
priority: evt.data("d").priority
}])
}
})
var action = $("<td></td>").addClass("dropdown")
var button = $("<button>Actions</button>").addClass("btn btn-secondary dropdown-toggle").attr("type", "button").attr("data-toggle", "dropdown")
var menu = $("<div></div>").addClass("dropdown-menu")
var del = $('<a href="#">Delete</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
if (confirm("confirm deletion of " + evt.data.download.filename)) {
deleteDownload([{
index: evt.data.index
}])
}
})
var resume = $('<a href="#">Resume</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
setDownload([{
index: evt.data.index,
status: "Queue",
priority: evt.data.download.priority
}])
})
var stop = $('<a href="#">Stop</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
setDownload([{
index: evt.data.index,
status: "Stop",
priority: evt.data.download.priority
}])
})
var na = $('<a href="#">N/A</a>').addClass("dropdown-item")
menu.append(del, resume, stop, na)
action.append(button, menu)
var progress = $("<td></td>").append(
$("<div></div>").addClass("progress").append(
$("<div></div>").addClass("progress-bar text-dark").css("width", (d.progress ?? "0") + '%').attr("id", "progress-" + i)
.text((d.progress ?? "0") + '%')
),
)
row.append(
index,
filename,
url,
status,
action,
progress
)
tbody.append(row)
i++;
}
$("#table").replaceWith(tbody)
}
function deleteDownload(data){
var xhr = new XMLHttpRequest();
xhr.open("DELETE", "http://gloader.narnian.us:8844/delete");
xhr.setRequestHeader("Content-Type", "application/javascript");
xhr.send(JSON.stringify(data));
}
function setDownload(data){
var xhr = new XMLHttpRequest();
xhr.open("POST", "http://gloader.narnian.us:8844/set");
xhr.setRequestHeader("Content-Type", "application/javascript");
xhr.send(JSON.stringify(data));
}

BIN
assets/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.3 KiB

9
assets/footer.tmpl Normal file
View File

@ -0,0 +1,9 @@
</div>
<script src="/jquery-3.2.1.slim.min.js"></script>
<script src="/popper.min.js"></script>
<script src="/bootstrap.min.js"></script>
<script src="/bootstrap-table.min.js"></script>
</html>

78
assets/header.tmpl Normal file
View File

@ -0,0 +1,78 @@
<!DOCTYPE html>
<html lang="en">
<title>{{ .Hostname }} — gloader</title>
<link rel="stylesheet" href="/bootstrap.min.css" />
<link rel="stylesheet" href="/bootstrap-table.min.css" />
<style type="text/css">
.progress-bar:nth-child(5n) {
background-color: #337ab7;
}
.progress-bar:nth-child(5n+1) {
background-color: #5cb85c;
}
.progress-bar:nth-child(5n+2) {
background-color: #5bc0de;
}
.progress-bar:nth-child(5n+3) {
background-color: #f0ad4e;
}
.progress-bar:nth-child(5n+4) {
background-color: #d9534f;
}
.lastlog {
text-overflow: ellipsis;
white-space: nowrap;
overflow: hidden;
}
.Error {
color: red;
}
.container-fluid {
padding-left: 45px;
}
.navbar-default .navbar-text {
color: #777;
}
.navbar-default {
background-color: #f8f8f8;
border-color: #e7e7e7;
}
body {
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
}
</style>
<nav class="navbar navbar-default">
<p style="margin-top: 0.25em; font-size: 18px"><a href="/">gokrazy</a><br>
<!-- <small style="font-size: 11px" class="text-muted">version { .BuildTimestamp }}</small> --></p>
<table class="navbar-text navbar-right" style="border-spacing: 10px 0; border-collapse: separate">
<tr>
<th>host</th>
<td>{{ .Hostname }}</td>
</tr>
<!-- <tr>
<th>kernel</th>
<td>{ .Kernel }}</td>
</tr>
{ if (ne .Model "") }}
<tr>
<th>model</th>
<td>{ .Model }}</td>
</tr>
{ end }}
{ if .EEPROM }}
<tr>
<th>EEPROM<br>(SHA256)</th>
<td>{ shortenSHA256 .EEPROM.PieepromSHA256 }}<br>{ shortenSHA256 .EEPROM.VL805SHA256 }}</td>
</tr>
{ end }} -->
</table>
</nav>
<div class="container-fluid">

52
assets/history.tmpl Normal file
View File

@ -0,0 +1,52 @@
{{ template "header" . }}
<div >
<!-- <h1>services</h1> -->
<table data-toggle="table" class="table table-hover">
<tbody id="table">
<thead>
<tr>
<th data-sortable="true" scope="col" width="5%">Index</th>
<th data-sortable="true" scope="col">Filename</th>
<th data-sortable="true" scope="col">URL</th>
<th data-sortable="true" scope="col">Status</th>
<th scope="col">Actions</th>
</tr>
</thead>
{{ range $idx, $request := .History }}
<tr class="download">
<th scope="row">{{ $idx }}</th>
<td class="lastlog">
{{ printf "%s/%s" $request.Subdir $request.Filename }}
</td>
<td>
{{ $request.URL }}
</td>
<td>
{{ printf "%v" $request.Status }}
{{ if $request.Error }}
{{ $request.Error }}
{{ end }}
</td>
<td class="dropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
<div class="dropdown-menu">
<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }}, history: true }])}">Delete</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority }} }])'>Stop</a>
<a class="dropdown-item" title="{{ printf "%s/%s" $request.Subdir $request.Filename }}" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
</div>
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
<script src="/downloads.js"></script>
{{ template "footer" . }}

4
assets/jquery-3.2.1.slim.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
assets/popper.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

70
assets/queue.tmpl Normal file
View File

@ -0,0 +1,70 @@
{{ template "header" . }}
<div >
<!-- <h1>services</h1> -->
<table data-toggle="table" class="table">
<thead>
<tr>
<th data-sortable="true" scope="col" width="5%">Index</th>
<th data-sortable="true" scope="col">Filename</th>
<th data-sortable="true" scope="col">URL</th>
<th data-sortable="true" scope="col">Status</th>
<th scope="col">Actions</th>
<th data-sortable="true" scope="col">Progress</th>
</tr>
</thead>
<tbody id="table">
{{ range $idx, $request := .Queue }}
<tr class="download">
<th scope="row">{{ $idx }}</th>
<td class="lastlog">
{{ printf "%s/%s" $request.Subdir $request.Filename }}
</td>
<td>
{{ $request.URL }}
</td>
<td>
{{ printf "%v" $request.Status }}
{{ if $request.Error }}
{{ $request.Error }}
{{ end }}
</td>
<td class="dropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
<div class="dropdown-menu">
<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }} }])}">Delete</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority | js}} }])'>Stop</a>
<a class="dropdown-item" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
</div>
</td>
<td>
<div class="progress">
<div id="progress-{{ $idx }}" class="progress-bar text-dark" style="width: 0{{ $request.Progress }}%">0{{ $request.Progress }}%</div>
</div>
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
<script src="/downloads.js"></script>
<script type="text/javascript">
function setDownloads() {
var xhr = new XMLHttpRequest();
xhr.open("GET", "http://gloader.narnian.us:8844/queued");
xhr.setRequestHeader("Accept", "application/javascript");
xhr.onload = evt => {
updateDownloads(JSON.parse(evt.target.responseText));
};
xhr.send();
}
var interval = setInterval(setDownloads, 5 * 1000)
</script>
{{ template "footer" . }}

3
bundle.go Normal file
View File

@ -0,0 +1,3 @@
package main
//go:generate sh -c "go run goembed.go -package bundled -var assets assets/history.tmpl assets/downloads.js assets/header.tmpl assets/footer.tmpl assets/queue.tmpl assets/status.tmpl assets/favicon.ico assets/bootstrap.min.css assets/bootstrap-table.min.css assets/bootstrap-table.min.js assets/bootstrap.min.js assets/popper.min.js assets/jquery-3.2.1.slim.min.js > bundled/GENERATED_bundled.go && gofmt -w bundled/GENERATED_bundled.go"

18
bundled/bundled.go Normal file
View File

@ -0,0 +1,18 @@
package bundled
import (
"bytes"
"net/http"
"time"
)
func Asset(basename string) string {
return string(assets["assets/"+basename])
}
func HTTPHandlerFunc(basename string) http.Handler {
modTime := time.Now()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, basename, modTime, bytes.NewReader(assets["assets/"+basename]))
})
}

481
downloader.go
View File

@ -1,11 +1,16 @@
package main package main
import ( import (
"bytes"
"context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"html/template"
"io"
"io/ioutil" "io/ioutil"
"log" "log"
"math"
"mime" "mime"
"net" "net"
"net/http" "net/http"
@ -19,21 +24,23 @@ import (
"strings" "strings"
"time" "time"
"git.narnian.us/lordwelch/gloader/bundled"
"github.com/cavaliercoder/grab" "github.com/cavaliercoder/grab"
"github.com/lordwelch/pathvalidate" "github.com/lordwelch/pathvalidate"
"github.com/wI2L/jettison"
"golang.org/x/net/publicsuffix" "golang.org/x/net/publicsuffix"
) )
var ( var (
DefaultCookieJar = newCookieJar()
DefaultGrabClient = grab.NewClient()
DefaultMaxActiveDownloads = 4 DefaultMaxActiveDownloads = 4
ErrUnsupportedScheme = errors.New("unsupported scheme") ErrUnsupportedScheme = errors.New("unsupported scheme")
) )
type Priority uint8 type Priority int
type Status uint8 type Status int
const ( const (
Highest Priority = iota Highest Priority = iota
@ -57,31 +64,33 @@ type Downloader struct {
DownloadDir string DownloadDir string
CompleteDir string CompleteDir string
InfoDir string InfoDir string
Grab *grab.Client
Jar http.CookieJar
MaxActiveDownloads int MaxActiveDownloads int
Server *http.Server Server *http.Server
Downloads RequestQueue Downloads RequestQueue
History RequestQueue History RequestQueue
NewRequest chan Request NewRequest chan Request
requestDone chan *Request requestDone chan *Request
OnComplete func(r Request) OnComplete func(d *Downloader, r Request)
OnAdd func(r Request) OnAdd func(d *Downloader, r Request)
} }
type Request struct { type Request struct {
URL string `json:"url"` URL string `json:"url"`
Cookies []http.Cookie `json:"cookies"` Cookies []http.Cookie `json:"cookies"`
ForceDownload bool `json:"forceDownload"` ForceDownload bool `json:"forceDownload"`
Status Status `json:"Status"` Status Status `json:"status"`
Priority Priority `json:"priority"` Priority Priority `json:"priority"`
FilePath string `json:"filepath"` FilePath string `json:"filepath"`
Filename string `json:"filename"` Filename string `json:"filename"`
Subdir string `json:"subdir"` Subdir string `json:"subdir"`
TempPath string `json:"tempPath"` TempPath string `json:"tempPath"`
Response *grab.Response `json:"-"` Response *grab.Response `json:"-"`
Error error `json:"-"` Error string `json:"error"`
Err error `json:"-"`
CompletedDate time.Time `json:"completedDate"` CompletedDate time.Time `json:"completedDate"`
Jar http.CookieJar `json:"-"`
Progress string `json:"progress,omitempty"`
grab *grab.Client
} }
type RequestQueue struct { type RequestQueue struct {
@ -90,6 +99,97 @@ type RequestQueue struct {
DateSort bool DateSort bool
} }
func (p Priority) MarshalJSON() ([]byte, error) {
var v string
switch p {
default:
v = "Medium"
case Medium:
v = "Medium"
case Low:
v = "Low"
case High:
v = "High"
case Highest:
v = "Highest"
}
return json.Marshal(v)
}
func (p *Priority) UnmarshalJSON(b []byte) error {
var (
v int
s string
)
if err := json.Unmarshal(b, &v); err == nil {
*p = Priority(v)
return nil
}
if err := json.Unmarshal(b, &s); err != nil {
return err
}
switch strings.ToLower(s) {
default:
*p = Medium
case "medium":
*p = Medium
case "low":
*p = Low
case "high":
*p = High
case "highest":
*p = Highest
}
return nil
}
func (s *Status) UnmarshalJSON(b []byte) error {
var v string
if err := json.Unmarshal(b, &v); err != nil {
return err
}
switch strings.ToLower(v) {
default:
*s = Queued
case "queued", "queue":
*s = Queued
case "complete", "completed":
*s = Complete
case "stop", "stopped":
*s = Stopped
case "download", "downloading":
*s = Downloading
case "error":
*s = Error
case "cancel", "canceled":
*s = Canceled
}
return nil
}
func (s Status) MarshalJSON() ([]byte, error) {
return json.Marshal(s.String())
}
func (s Status) String() string {
switch s {
default:
return "Queued"
case Queued:
return "Queued"
case Complete:
return "Complete"
case Stopped:
return "Stopped"
case Downloading:
return "Downloading"
case Error:
return "Error"
case Canceled:
return "Canceled"
}
}
func (rq RequestQueue) Less(i, j int) bool { func (rq RequestQueue) Less(i, j int) bool {
ii := 0 ii := 0
jj := 0 jj := 0
@ -145,18 +245,47 @@ func (rq *RequestQueue) remove(r *Request) {
} }
} }
func (rq *RequestQueue) updateStatus() {
for _, req := range rq.Queue {
if req.Response != nil {
req.Progress = fmt.Sprintf("%.2f", math.Abs(req.Response.Progress()*100))
}
}
}
func (r *Request) setError(err error) {
if err != nil {
r.Status = Error
r.Err = err
r.Error = err.Error()
} else {
r.Status = Paused
r.Err = nil
r.Error = ""
}
}
func (r Request) Delete() error {
var err, ret error
if r.Response != nil {
err = r.Response.Cancel()
if err != nil && err != context.Canceled {
ret = err
}
}
_ = os.Remove(r.TempPath)
err = os.Remove(r.FilePath)
if err != nil && err != os.ErrNotExist {
ret = err
}
return ret
}
func newCookieJar() http.CookieJar { func newCookieJar() http.CookieJar {
c, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) c, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
return c return c
} }
func newDownloader() *Downloader {
return &Downloader{
Jar: DefaultCookieJar,
Grab: DefaultGrabClient,
}
}
func (d *Downloader) Start(network, address string) { func (d *Downloader) Start(network, address string) {
var ( var (
listener net.Listener listener net.Listener
@ -183,7 +312,6 @@ func (d *Downloader) Start(network, address string) {
ReadTimeout: 2 * time.Minute, ReadTimeout: 2 * time.Minute,
WriteTimeout: 2 * time.Minute, WriteTimeout: 2 * time.Minute,
} }
} }
if d.DataDir == "" { if d.DataDir == "" {
@ -209,26 +337,17 @@ func (d *Downloader) Start(network, address string) {
if err != nil { if err != nil {
panic(err) panic(err)
} }
log.Println("adding /add handler") log.Println("adding http handlers")
// mux.HandleFunc("/", d.UI)
d.initStatus(mux)
mux.HandleFunc("/add", d.restAddDownload) mux.HandleFunc("/add", d.restAddDownload)
mux.HandleFunc("/queue", d.restQueueStatus) mux.HandleFunc("/queued", d.restStatus(true))
mux.HandleFunc("/history", d.restHistoryStatus) mux.HandleFunc("/completed", d.restStatus(false))
mux.HandleFunc("/start", d.restStartDownload) mux.HandleFunc("/set", d.restSetDownloadStatus)
mux.HandleFunc("/delete", d.restDelete)
mux.Handle("/get/", http.StripPrefix("/get/", http.FileServer(http.Dir(d.CompleteDir))))
log.Println("starting main go routine") log.Println("starting main go routine")
d.Grab.HTTPClient = &http.Client{
Jar: d.Jar,
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
}
go d.download() go d.download()
@ -236,64 +355,117 @@ func (d *Downloader) Start(network, address string) {
_ = d.Server.Serve(listener) _ = d.Server.Serve(listener)
} }
func (d *Downloader) restStartDownload(w http.ResponseWriter, r *http.Request) { func httpMethodNotAllowed(w http.ResponseWriter, r *http.Request, method string) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", method)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintf(w, "HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
log.Printf("HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
}
func (d *Downloader) restDelete(w http.ResponseWriter, r *http.Request) {
var ( var (
err error err error
index struct { index []struct {
index int Index int
History bool
} }
ret []Request
) )
if r.Method != http.MethodPost { if r.Method != http.MethodDelete {
w.Header().Set("Content-Type", "text/plain; charset=utf-8") httpMethodNotAllowed(w, r, http.MethodDelete)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodPost)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
return return
} }
err = json.NewDecoder(r.Body).Decode(index) err = json.NewDecoder(r.Body).Decode(&index)
if err != nil { if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
if index.index >= d.Downloads.Len() || index.index < 0 { sort.Slice(index, func(i, j int) bool {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", index.index, d.Downloads.Len()), http.StatusBadRequest) return index[i].Index < index[j].Index
return })
for x, i := range index {
if i.History {
if i.Index >= d.History.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
ret = append(ret, *d.History.Pop(i.Index - x))
ret[len(ret)-1].Delete()
} else {
if i.Index >= d.Downloads.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
ret = append(ret, *d.Downloads.Pop(i.Index - x))
ret[len(ret)-1].Delete()
}
} }
d.startDownload(index.index) d.syncDownloads()
}
func (d *Downloader) restHistoryStatus(w http.ResponseWriter, r *http.Request) { v, err := jettison.MarshalOpts(ret, jettison.DenyList([]string{"cookies"}))
if r.Method != http.MethodGet { if err != nil {
w.Header().Set("Content-Type", "text/plain; charset=utf-8") http.Error(w, err.Error(), http.StatusInternalServerError)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodGet)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
return return
} }
j := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json; charset=utf-8") w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
j.Encode(d.History.Queue) w.Write(v)
} }
func (d *Downloader) restQueueStatus(w http.ResponseWriter, r *http.Request) { func (d *Downloader) restSetDownloadStatus(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet { var (
w.Header().Set("Content-Type", "text/plain; charset=utf-8") err error
w.Header().Set("X-Content-Type-Options", "nosniff") index []struct {
w.Header().Add("Allow", http.MethodGet) Index int
w.WriteHeader(http.StatusMethodNotAllowed) Status Status
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed") Priority Priority
log.Println("HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed") }
)
if r.Method != http.MethodPost {
httpMethodNotAllowed(w, r, http.MethodPost)
return return
} }
j := json.NewEncoder(w) err = json.NewDecoder(r.Body).Decode(&index)
w.Header().Add("Content-Type", "application/json; charset=utf-8") if err != nil {
w.WriteHeader(http.StatusOK) http.Error(w, err.Error(), http.StatusBadRequest)
j.Encode(d.Downloads.Queue) return
}
for _, i := range index {
if i.Index >= d.Downloads.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
r := d.Downloads.Queue[i.Index]
r.Priority = i.Priority
r.Status = i.Status
}
d.syncDownloads()
}
func (d *Downloader) restStatus(q bool) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
var queue = &d.History
if q {
d.Downloads.updateStatus()
queue = &d.Downloads
}
if r.Method != http.MethodGet {
httpMethodNotAllowed(w, r, http.MethodGet)
return
}
queue.updateStatus()
v, err := jettison.MarshalOpts(queue.Queue, jettison.DenyList([]string{"cookies"}))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write(v)
}
} }
func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) { func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
@ -302,12 +474,7 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
err error err error
) )
if r.Method != http.MethodPost { if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "text/plain; charset=utf-8") httpMethodNotAllowed(w, r, http.MethodPost)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodPost)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
return return
} }
// TODO fail only on individual requests // TODO fail only on individual requests
@ -334,8 +501,10 @@ func (d Downloader) getNameFromHEAD(r Request) string {
re *http.Response re *http.Response
p map[string]string p map[string]string
) )
r.insertCookies()
ht := &http.Client{ ht := &http.Client{
Jar: d.Jar, Jar: r.Jar,
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
Transport: &http.Transport{ Transport: &http.Transport{
Dial: (&net.Dialer{ Dial: (&net.Dialer{
@ -464,14 +633,6 @@ func (d Downloader) FindRequest(u string) *Request {
func (d *Downloader) addRequest(r *Request) { func (d *Downloader) addRequest(r *Request) {
log.Println("adding download for", r.URL) log.Println("adding download for", r.URL)
req := d.FindRequest(r.URL) req := d.FindRequest(r.URL)
u, _ := url.Parse(r.URL)
for i, v := range r.Cookies {
d.Jar.SetCookies(&url.URL{
Scheme: u.Scheme,
Path: v.Path,
Host: v.Domain,
}, []*http.Cookie{&r.Cookies[i]})
}
d.getFilename(r) d.getFilename(r)
if req != nil { // url alread added if req != nil { // url alread added
@ -502,20 +663,37 @@ func (d *Downloader) startDownload(i int) {
err error err error
) )
r = d.Downloads.Queue[i] r = d.Downloads.Queue[i]
r.insertCookies()
d.getTempFilename(r) d.getTempFilename(r)
log.Println("starting download for", r.URL, "to", r.TempPath) log.Println("starting download for", r.URL, "to", r.TempPath)
// d.Downloads.Queue = append(d.Downloads.Queue, r) // d.Downloads.Queue = append(d.Downloads.Queue, r)
if r.Response == nil || r.Response.Err() != nil { if r.Response == nil || r.Response.Err() != nil {
req, err = grab.NewRequest(r.TempPath, r.URL) req, err = grab.NewRequest(r.TempPath, r.URL)
if err != nil { if err != nil {
r.Status = Error r.setError(err)
r.Error = err
return return
} }
} }
r.Status = Downloading r.Status = Downloading
r.Response = d.Grab.Do(req) if r.grab == nil {
r.grab = &grab.Client{
HTTPClient: &http.Client{
Jar: r.Jar,
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
},
}
}
r.Response = r.grab.Do(req)
go func(r *Request) { go func(r *Request) {
log.Println("wait for download") log.Println("wait for download")
log.Println(r.Response.IsComplete()) log.Println(r.Response.IsComplete())
@ -542,12 +720,43 @@ func (d *Downloader) syncDownloads() {
return return
} }
sort.Stable(d.Downloads) sort.Stable(d.Downloads)
var downloadsMaxed bool
// Start new downloads // Start new downloads
for i, req := range d.Downloads.Queue { for i, req := range d.Downloads.Queue {
if d.MaxActiveDownloads >= len(d.getRunningDownloads()) { switch req.Status {
if req.Status == Queued { case Queued:
if !downloadsMaxed && d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
d.startDownload(i) d.startDownload(i)
} }
case Stopped:
if req.Response != nil {
var err = req.Response.Cancel()
if err != nil && err != context.Canceled {
req.setError(err)
}
req.Response = nil
}
case Downloading:
if req.Response == nil {
d.startDownload(i)
}
case Error:
if req.Response != nil && req.Err == nil {
var err = req.Response.Err()
var err2 = req.Response.Cancel()
if err != nil && err2 != context.Canceled {
err = err2
}
req.setError(err)
}
case Canceled:
if req.Response != nil {
err := req.Response.Cancel()
if err != nil && err != context.Canceled {
req.setError(err)
}
req.Response = nil
}
} }
} }
@ -558,6 +767,7 @@ func (d *Downloader) syncDownloads() {
i-- i--
} }
} }
d.Downloads.updateStatus()
} }
func (d *Downloader) requestCompleted(r *Request) { func (d *Downloader) requestCompleted(r *Request) {
@ -572,9 +782,8 @@ func (d *Downloader) requestCompleted(r *Request) {
} }
d.History.Queue = append(d.History.Queue, r) d.History.Queue = append(d.History.Queue, r)
} else { } else {
r.Status = Error r.setError(r.Response.Err())
r.Error = r.Response.Err() log.Println("fucking error:", r.Err)
log.Println("fucking error:", r.Error)
} }
} }
@ -585,17 +794,93 @@ func (d *Downloader) download() {
d.syncDownloads() d.syncDownloads()
case r := <-d.NewRequest: case r := <-d.NewRequest:
r.Progress = "" // not allowed when adding
d.addRequest(&r) d.addRequest(&r)
if d.OnAdd != nil { if d.OnAdd != nil {
d.OnAdd(r) d.OnAdd(d, r)
} }
case r := <-d.requestDone: case r := <-d.requestDone:
log.Println("finishing request for", r.URL) log.Println("finishing request for", r.URL)
d.requestCompleted(r) d.requestCompleted(r)
if d.OnComplete != nil { if d.OnComplete != nil {
d.OnComplete(*r) d.OnComplete(d, *r)
} }
} }
} }
} }
func (r *Request) insertCookies() {
if r.Jar == nil {
r.Jar = newCookieJar()
}
u, _ := url.Parse(r.URL)
for i, v := range r.Cookies {
r.Jar.SetCookies(&url.URL{
Scheme: u.Scheme,
Path: v.Path,
Host: v.Domain,
}, []*http.Cookie{&r.Cookies[i]})
}
}
func (d *Downloader) initStatus(mux *http.ServeMux) {
commonTmpls := template.New("root").Funcs(map[string]interface{}{
"shortenSHA256": func(hash string) string {
if len(hash) > 10 {
return hash[:10]
}
return hash
},
})
commonTmpls = template.Must(commonTmpls.New("header").Parse(bundled.Asset("header.tmpl")))
commonTmpls = template.Must(commonTmpls.New("footer").Parse(bundled.Asset("footer.tmpl")))
queueTmpl := template.Must(template.Must(commonTmpls.Clone()).New("queueTmpl").Parse(bundled.Asset("queue.tmpl")))
historyTmpl := template.Must(template.Must(commonTmpls.Clone()).New("historyTmpl").Parse(bundled.Asset("history.tmpl")))
for _, fn := range []string{
"favicon.ico",
"bootstrap.min.js",
"bootstrap.min.css",
"bootstrap-table.min.js",
"bootstrap-table.min.css",
"popper.min.js",
"jquery-3.2.1.slim.min.js",
"downloads.js",
} {
mux.Handle("/"+fn, bundled.HTTPHandlerFunc(fn))
}
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
d.Downloads.updateStatus()
if err := queueTmpl.Execute(&buf, struct {
Queue []*Request
History []*Request
Hostname string
}{
Queue: d.Downloads.Queue,
History: d.History.Queue,
Hostname: "gloader",
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
io.Copy(w, &buf)
})
mux.HandleFunc("/history", func(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
d.Downloads.updateStatus()
if err := historyTmpl.Execute(&buf, struct {
Queue []*Request
History []*Request
Hostname string
}{
Queue: d.Downloads.Queue,
History: d.History.Queue,
Hostname: "gloader",
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
io.Copy(w, &buf)
})
}

1
go.mod
View File

@ -8,5 +8,6 @@ require (
github.com/cavaliercoder/grab v2.0.0+incompatible github.com/cavaliercoder/grab v2.0.0+incompatible
github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308 github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308
github.com/u-root/u-root v7.0.0+incompatible github.com/u-root/u-root v7.0.0+incompatible
github.com/wI2L/jettison v0.7.1
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
) )

187
goembed.go Normal file
View File

@ -0,0 +1,187 @@
// +build ignore
// goembed generates a Go source file from an input file.
package main
import (
"bufio"
"bytes"
"compress/gzip"
"flag"
"fmt"
"io"
"log"
"os"
"text/template"
"unicode/utf8"
)
var (
packageFlag = flag.String("package", "", "Go package name")
varFlag = flag.String("var", "", "Go var name")
gzipFlag = flag.Bool("gzip", false, "Whether to gzip contents")
)
func main() {
flag.Parse()
fmt.Printf("package %s\n\n", *packageFlag)
if *gzipFlag {
err := gzipPrologue.Execute(os.Stdout, map[string]interface{}{
"Args": flag.Args(),
"VarName": *varFlag,
})
if err != nil {
log.Fatal(err)
}
}
if flag.NArg() > 0 {
fmt.Println("// Table of contents")
fmt.Printf("var %v = map[string][]byte{\n", *varFlag)
for i, filename := range flag.Args() {
fmt.Printf("\t%q: %s_%d,\n", filename, *varFlag, i)
}
fmt.Println("}")
// Using a separate variable for each []byte, instead of
// combining them into a single map literal, enables a storage
// optimization: the compiler places the data directly in the
// program's noptrdata section instead of the heap.
for i, filename := range flag.Args() {
if err := oneVar(fmt.Sprintf("%s_%d", *varFlag, i), filename); err != nil {
log.Fatal(err)
}
}
} else {
if err := oneVarReader(*varFlag, os.Stdin); err != nil {
log.Fatal(err)
}
}
}
func oneVar(varName, filename string) error {
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
return oneVarReader(varName, f)
}
func oneVarReader(varName string, r io.Reader) error {
// Generate []byte(<big string constant>) instead of []byte{<list of byte values>}.
// The latter causes a memory explosion in the compiler (60 MB of input chews over 9 GB RAM).
// Doing a string conversion avoids some of that, but incurs a slight startup cost.
if !*gzipFlag {
fmt.Printf(`var %s = []byte("`, varName)
} else {
var buf bytes.Buffer
gzw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
if _, err := io.Copy(gzw, r); err != nil {
return err
}
if err := gzw.Close(); err != nil {
return err
}
fmt.Printf("var %s []byte // set in init\n\n", varName)
fmt.Printf(`var %s_gzip = []byte("`, varName)
r = &buf
}
bufw := bufio.NewWriter(os.Stdout)
if _, err := io.Copy(&writer{w: bufw}, r); err != nil {
return err
}
if err := bufw.Flush(); err != nil {
return err
}
fmt.Println(`")`)
return nil
}
type writer struct {
w io.Writer
}
func (w *writer) Write(data []byte) (n int, err error) {
n = len(data)
for err == nil && len(data) > 0 {
// https://golang.org/ref/spec#String_literals: "Within the quotes, any
// character may appear except newline and unescaped double quote. The
// text between the quotes forms the value of the literal, with backslash
// escapes interpreted as they are in rune literals […]."
switch b := data[0]; b {
case '\\':
_, err = w.w.Write([]byte(`\\`))
case '"':
_, err = w.w.Write([]byte(`\"`))
case '\n':
_, err = w.w.Write([]byte(`\n`))
case '\x00':
// https://golang.org/ref/spec#Source_code_representation: "Implementation
// restriction: For compatibility with other tools, a compiler may
// disallow the NUL character (U+0000) in the source text."
_, err = w.w.Write([]byte(`\x00`))
default:
// https://golang.org/ref/spec#Source_code_representation: "Implementation
// restriction: […] A byte order mark may be disallowed anywhere else in
// the source."
const byteOrderMark = '\uFEFF'
if r, size := utf8.DecodeRune(data); r != utf8.RuneError && r != byteOrderMark {
_, err = w.w.Write(data[:size])
data = data[size:]
continue
}
_, err = fmt.Fprintf(w.w, `\x%02x`, b)
}
data = data[1:]
}
return n - len(data), err
}
var gzipPrologue = template.Must(template.New("").Parse(`
import (
"bytes"
"compress/gzip"
"io/ioutil"
)
func init() {
var (
r *gzip.Reader
err error
)
{{ if gt (len .Args) 0 }}
{{ range $idx, $var := .Args }}
{{ $n := printf "%s_%d" $.VarName $idx }}
r, err = gzip.NewReader(bytes.NewReader({{ $n }}_gzip))
if err != nil {
panic(err)
}
{{ $n }}, err = ioutil.ReadAll(r)
{{ $.VarName }}["{{ $var }}"] = {{ $n }}
r.Close()
if err != nil {
panic(err)
}
{{ end }}
{{ else }}
r, err = gzip.NewReader(bytes.NewReader({{ .VarName }}_gzip))
if err != nil {
panic(err)
}
{{ .VarName }}, err = ioutil.ReadAll(r)
r.Close()
if err != nil {
panic(err)
}
{{ end }}
}
`))

3
grab/client.go
View File

@ -267,6 +267,7 @@ func (c *Client) validateLocal(resp *Response) stateFunc {
if expectedSize >= 0 && expectedSize < resp.fi.Size() { if expectedSize >= 0 && expectedSize < resp.fi.Size() {
// remote size is known, is smaller than local size and we want to resume // remote size is known, is smaller than local size and we want to resume
fmt.Fprintln(os.Stderr, "validate\n")
resp.err = ErrBadLength resp.err = ErrBadLength
return c.closeResponse return c.closeResponse
} }
@ -395,6 +396,7 @@ func (c *Client) readResponse(resp *Response) stateFunc {
// remote size is known // remote size is known
resp.sizeUnsafe += resp.bytesResumed resp.sizeUnsafe += resp.bytesResumed
if resp.Request.Size > 0 && resp.Request.Size != resp.sizeUnsafe { if resp.Request.Size > 0 && resp.Request.Size != resp.sizeUnsafe {
fmt.Fprintln(os.Stderr, "response\n")
resp.err = ErrBadLength resp.err = ErrBadLength
return c.closeResponse return c.closeResponse
} }
@ -527,6 +529,7 @@ func (c *Client) copyFile(resp *Response) stateFunc {
discoveredSize := resp.bytesResumed + bytesCopied discoveredSize := resp.bytesResumed + bytesCopied
atomic.StoreInt64(&resp.sizeUnsafe, discoveredSize) atomic.StoreInt64(&resp.sizeUnsafe, discoveredSize)
if resp.Request.Size > 0 && resp.Request.Size != discoveredSize { if resp.Request.Size > 0 && resp.Request.Size != discoveredSize {
fmt.Fprintln(os.Stderr, "file\n")
resp.err = ErrBadLength resp.err = ErrBadLength
return c.closeResponse return c.closeResponse
} }

66
main.go
View File

@ -32,40 +32,42 @@ func main() {
log.Println(err) log.Println(err)
os.Exit(1) os.Exit(1)
} }
d := newDownloader() d := &Downloader{
loadQueue(d) OnAdd: save,
save := func(r Request) { OnComplete: save,
var ( DataDir: filepath.Join(gloaderHome, "data"),
content []byte
err error
)
content, err = json.Marshal(d.History.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "history.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
content, err = json.Marshal(d.Downloads.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "queue.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
} }
d.OnAdd = save loadQueue(d)
d.OnComplete = save
d.DataDir = filepath.Join(gloaderHome, "data")
d.Start("tcp", ":8844") d.Start("tcp", ":8844")
} }
func save(d *Downloader, r Request) {
var (
content []byte
err error
)
content, err = json.Marshal(d.History.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "history.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
content, err = json.Marshal(d.Downloads.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "queue.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
}
func loadQueue(d *Downloader) { func loadQueue(d *Downloader) {
var ( var (
f io.ReadCloser f io.ReadCloser
@ -142,6 +144,10 @@ func mount() error {
if err != nil { if err != nil {
return fmt.Errorf("error mounting datadir: %w", err) return fmt.Errorf("error mounting datadir: %w", err)
} }
err = syscall.Mount(dev.Path, dataDir, "ext4", syscall.MS_SHARED|syscall.MS_REMOUNT, "")
if err != nil {
return fmt.Errorf("error remounting datadir: %w", err)
}
return nil return nil
} }
return fmt.Errorf("error mounting datadir: %w", err) return fmt.Errorf("error mounting datadir: %w", err)