lordwelch/gloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor cookie handling

Browse Source 
Switch to jettison for JSON marshaling
Create an HTTP UI
Use goembed for assets
This commit is contained in:
lordwelch 2020-12-21 01:15:16 -08:00
parent bd0c3d2cc6
commit 36e79aa907
19 changed files with 968 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
assets/bootstrap-table.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

10
assets/bootstrap-table.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

7
assets/bootstrap.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

7
assets/bootstrap.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

85
assets/downloads.js Normal file
View File

@ -0,0 +1,85 @@
function updateDownloads(downloads) {
var dls = [];
var i = 0;
var tbody = $("<tbody></tbody>").attr("id", "table")
for (var d of downloads) {
var row = $("<tr></tr>").addClass("download").attr("index", i)
var index = $("<th></th>").attr("scope", "row").text(i)
var filename = $("<td></td>").text(d.subdir + "/" + d.filename).addClass("lastlog")
var url = $("<td></td>").text(d.url)
var status = $("<td></td>").addClass(d.status).css("cursor", "pointer").text(d.status + "\n" + d.error).on("click",{index: i, download: d}, evt => {
if (d.status != "Queue" || d.status != "Complete") {
setDownload([{
index: evt.data("index"),
status: "Queue",
priority: evt.data("d").priority
}])
}
})
var action = $("<td></td>").addClass("dropdown")
var button = $("<button>Actions</button>").addClass("btn btn-secondary dropdown-toggle").attr("type", "button").attr("data-toggle", "dropdown")
var menu = $("<div></div>").addClass("dropdown-menu")
var del = $('<a href="#">Delete</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
if (confirm("confirm deletion of " + evt.data.download.filename)) {
deleteDownload([{
index: evt.data.index
}])
}
})
var resume = $('<a href="#">Resume</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
setDownload([{
index: evt.data.index,
status: "Queue",
priority: evt.data.download.priority
}])
})
var stop = $('<a href="#">Stop</a>').addClass("dropdown-item").on("click",{index: i, download: d}, evt => {
setDownload([{
index: evt.data.index,
status: "Stop",
priority: evt.data.download.priority
}])
})
var na = $('<a href="#">N/A</a>').addClass("dropdown-item")
menu.append(del, resume, stop, na)
action.append(button, menu)
var progress = $("<td></td>").append(
$("<div></div>").addClass("progress").append(
$("<div></div>").addClass("progress-bar text-dark").css("width", (d.progress ?? "0") + '%').attr("id", "progress-" + i)
.text((d.progress ?? "0") + '%')
),
)
row.append(
index,
filename,
url,
status,
action,
progress
)
tbody.append(row)
i++;
}
$("#table").replaceWith(tbody)
}
function deleteDownload(data){
var xhr = new XMLHttpRequest();
xhr.open("DELETE", "http://gloader.narnian.us:8844/delete");
xhr.setRequestHeader("Content-Type", "application/javascript");
xhr.send(JSON.stringify(data));
}
function setDownload(data){
var xhr = new XMLHttpRequest();
xhr.open("POST", "http://gloader.narnian.us:8844/set");
xhr.setRequestHeader("Content-Type", "application/javascript");
xhr.send(JSON.stringify(data));
}

BIN
assets/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.3 KiB

9
assets/footer.tmpl Normal file
View File

@ -0,0 +1,9 @@
</div>
<script src="/jquery-3.2.1.slim.min.js"></script>
<script src="/popper.min.js"></script>
<script src="/bootstrap.min.js"></script>
<script src="/bootstrap-table.min.js"></script>
</html>

78
assets/header.tmpl Normal file
View File

@ -0,0 +1,78 @@
<!DOCTYPE html>
<html lang="en">
<title>{{ .Hostname }} — gloader</title>
<link rel="stylesheet" href="/bootstrap.min.css" />
<link rel="stylesheet" href="/bootstrap-table.min.css" />
<style type="text/css">
.progress-bar:nth-child(5n) {
background-color: #337ab7;
}
.progress-bar:nth-child(5n+1) {
background-color: #5cb85c;
}
.progress-bar:nth-child(5n+2) {
background-color: #5bc0de;
}
.progress-bar:nth-child(5n+3) {
background-color: #f0ad4e;
}
.progress-bar:nth-child(5n+4) {
background-color: #d9534f;
}
.lastlog {
text-overflow: ellipsis;
white-space: nowrap;
overflow: hidden;
}
.Error {
color: red;
}
.container-fluid {
padding-left: 45px;
}
.navbar-default .navbar-text {
color: #777;
}
.navbar-default {
background-color: #f8f8f8;
border-color: #e7e7e7;
}
body {
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
}
</style>
<nav class="navbar navbar-default">
<p style="margin-top: 0.25em; font-size: 18px"><a href="/">gokrazy</a><br>
<!-- <small style="font-size: 11px" class="text-muted">version { .BuildTimestamp }}</small> --></p>
<table class="navbar-text navbar-right" style="border-spacing: 10px 0; border-collapse: separate">
<tr>
<th>host</th>
<td>{{ .Hostname }}</td>
</tr>
<!-- <tr>
<th>kernel</th>
<td>{ .Kernel }}</td>
</tr>
{ if (ne .Model "") }}
<tr>
<th>model</th>
<td>{ .Model }}</td>
</tr>
{ end }}
{ if .EEPROM }}
<tr>
<th>EEPROM<br>(SHA256)</th>
<td>{ shortenSHA256 .EEPROM.PieepromSHA256 }}<br>{ shortenSHA256 .EEPROM.VL805SHA256 }}</td>
</tr>
{ end }} -->
</table>
</nav>
<div class="container-fluid">

52
assets/history.tmpl Normal file
View File

@ -0,0 +1,52 @@
{{ template "header" . }}
<div >
<!-- <h1>services</h1> -->
<table data-toggle="table" class="table table-hover">
<tbody id="table">
<thead>
<tr>
<th data-sortable="true" scope="col" width="5%">Index</th>
<th data-sortable="true" scope="col">Filename</th>
<th data-sortable="true" scope="col">URL</th>
<th data-sortable="true" scope="col">Status</th>
<th scope="col">Actions</th>
</tr>
</thead>
{{ range $idx, $request := .History }}
<tr class="download">
<th scope="row">{{ $idx }}</th>
<td class="lastlog">
{{ printf "%s/%s" $request.Subdir $request.Filename }}
</td>
<td>
{{ $request.URL }}
</td>
<td>
{{ printf "%v" $request.Status }}
{{ if $request.Error }}
{{ $request.Error }}
{{ end }}
</td>
<td class="dropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
<div class="dropdown-menu">
<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }}, history: true }])}">Delete</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority }} }])'>Stop</a>
<a class="dropdown-item" title="{{ printf "%s/%s" $request.Subdir $request.Filename }}" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
</div>
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
<script src="/downloads.js"></script>
{{ template "footer" . }}

4
assets/jquery-3.2.1.slim.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
assets/popper.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

70
assets/queue.tmpl Normal file
View File

@ -0,0 +1,70 @@
{{ template "header" . }}
<div >
<!-- <h1>services</h1> -->
<table data-toggle="table" class="table">
<thead>
<tr>
<th data-sortable="true" scope="col" width="5%">Index</th>
<th data-sortable="true" scope="col">Filename</th>
<th data-sortable="true" scope="col">URL</th>
<th data-sortable="true" scope="col">Status</th>
<th scope="col">Actions</th>
<th data-sortable="true" scope="col">Progress</th>
</tr>
</thead>
<tbody id="table">
{{ range $idx, $request := .Queue }}
<tr class="download">
<th scope="row">{{ $idx }}</th>
<td class="lastlog">
{{ printf "%s/%s" $request.Subdir $request.Filename }}
</td>
<td>
{{ $request.URL }}
</td>
<td>
{{ printf "%v" $request.Status }}
{{ if $request.Error }}
{{ $request.Error }}
{{ end }}
</td>
<td class="dropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" data-toggle="dropdown">Actions</button>
<div class="dropdown-menu">
<a class="dropdown-item" href="#" onclick="if (confirm('confirm deletion of ' + {{ printf "%s/%s" $request.Subdir $request.Filename }})) { deleteDownload([{index: {{ $idx }} }])}">Delete</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Queue", priority: {{ $request.Priority }} }])'>Resume</a>
<a class="dropdown-item" href="#" onclick='setDownload([{ index: {{ $idx }}, status: "Stop", priority: {{ $request.Priority | js}} }])'>Stop</a>
<a class="dropdown-item" href="/get/{{ printf "%s/%s" $request.Subdir $request.Filename }}">Retrieve</a>
</div>
</td>
<td>
<div class="progress">
<div id="progress-{{ $idx }}" class="progress-bar text-dark" style="width: 0{{ $request.Progress }}%">0{{ $request.Progress }}%</div>
</div>
</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
<script src="/downloads.js"></script>
<script type="text/javascript">
function setDownloads() {
var xhr = new XMLHttpRequest();
xhr.open("GET", "http://gloader.narnian.us:8844/queued");
xhr.setRequestHeader("Accept", "application/javascript");
xhr.onload = evt => {
updateDownloads(JSON.parse(evt.target.responseText));
};
xhr.send();
}
var interval = setInterval(setDownloads, 5 * 1000)
</script>
{{ template "footer" . }}

3
bundle.go Normal file
View File

@ -0,0 +1,3 @@
package main
//go:generate sh -c "go run goembed.go -package bundled -var assets assets/history.tmpl assets/downloads.js assets/header.tmpl assets/footer.tmpl assets/queue.tmpl assets/status.tmpl assets/favicon.ico assets/bootstrap.min.css assets/bootstrap-table.min.css assets/bootstrap-table.min.js assets/bootstrap.min.js assets/popper.min.js assets/jquery-3.2.1.slim.min.js > bundled/GENERATED_bundled.go && gofmt -w bundled/GENERATED_bundled.go"

18
bundled/bundled.go Normal file
View File

@ -0,0 +1,18 @@
package bundled
import (
"bytes"
"net/http"
"time"
)
func Asset(basename string) string {
return string(assets["assets/"+basename])
}
func HTTPHandlerFunc(basename string) http.Handler {
modTime := time.Now()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, basename, modTime, bytes.NewReader(assets["assets/"+basename]))
})
}

481
downloader.go
View File

@ -1,11 +1,16 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"io/ioutil"
"log"
"math"
"mime"
"net"
"net/http"
@ -19,21 +24,23 @@ import (
"strings"
"time"
"git.narnian.us/lordwelch/gloader/bundled"
"github.com/cavaliercoder/grab"
"github.com/lordwelch/pathvalidate"
"github.com/wI2L/jettison"
"golang.org/x/net/publicsuffix"
)
var (
DefaultCookieJar = newCookieJar()
DefaultGrabClient = grab.NewClient()
DefaultMaxActiveDownloads = 4
ErrUnsupportedScheme = errors.New("unsupported scheme")
)
type Priority uint8
type Status uint8
type Priority int
type Status int
const (
Highest Priority = iota
@ -57,31 +64,33 @@ type Downloader struct {
DownloadDir string
CompleteDir string
InfoDir string
Grab *grab.Client
Jar http.CookieJar
MaxActiveDownloads int
Server *http.Server
Downloads RequestQueue
History RequestQueue
NewRequest chan Request
requestDone chan *Request
OnComplete func(r Request)
OnAdd func(r Request)
OnComplete func(d *Downloader, r Request)
OnAdd func(d *Downloader, r Request)
}
type Request struct {
URL string `json:"url"`
Cookies []http.Cookie `json:"cookies"`
ForceDownload bool `json:"forceDownload"`
Status Status `json:"Status"`
Status Status `json:"status"`
Priority Priority `json:"priority"`
FilePath string `json:"filepath"`
Filename string `json:"filename"`
Subdir string `json:"subdir"`
TempPath string `json:"tempPath"`
Response *grab.Response `json:"-"`
Error error `json:"-"`
Error string `json:"error"`
Err error `json:"-"`
CompletedDate time.Time `json:"completedDate"`
Jar http.CookieJar `json:"-"`
Progress string `json:"progress,omitempty"`
grab *grab.Client
}
type RequestQueue struct {
@ -90,6 +99,97 @@ type RequestQueue struct {
DateSort bool
}
func (p Priority) MarshalJSON() ([]byte, error) {
var v string
switch p {
default:
v = "Medium"
case Medium:
v = "Medium"
case Low:
v = "Low"
case High:
v = "High"
case Highest:
v = "Highest"
}
return json.Marshal(v)
}
func (p *Priority) UnmarshalJSON(b []byte) error {
var (
v int
s string
)
if err := json.Unmarshal(b, &v); err == nil {
*p = Priority(v)
return nil
}
if err := json.Unmarshal(b, &s); err != nil {
return err
}
switch strings.ToLower(s) {
default:
*p = Medium
case "medium":
*p = Medium
case "low":
*p = Low
case "high":
*p = High
case "highest":
*p = Highest
}
return nil
}
func (s *Status) UnmarshalJSON(b []byte) error {
var v string
if err := json.Unmarshal(b, &v); err != nil {
return err
}
switch strings.ToLower(v) {
default:
*s = Queued
case "queued", "queue":
*s = Queued
case "complete", "completed":
*s = Complete
case "stop", "stopped":
*s = Stopped
case "download", "downloading":
*s = Downloading
case "error":
*s = Error
case "cancel", "canceled":
*s = Canceled
}
return nil
}
func (s Status) MarshalJSON() ([]byte, error) {
return json.Marshal(s.String())
}
func (s Status) String() string {
switch s {
default:
return "Queued"
case Queued:
return "Queued"
case Complete:
return "Complete"
case Stopped:
return "Stopped"
case Downloading:
return "Downloading"
case Error:
return "Error"
case Canceled:
return "Canceled"
}
}
func (rq RequestQueue) Less(i, j int) bool {
ii := 0
jj := 0
@ -145,18 +245,47 @@ func (rq *RequestQueue) remove(r *Request) {
}
}
func (rq *RequestQueue) updateStatus() {
for _, req := range rq.Queue {
if req.Response != nil {
req.Progress = fmt.Sprintf("%.2f", math.Abs(req.Response.Progress()*100))
}
}
}
func (r *Request) setError(err error) {
if err != nil {
r.Status = Error
r.Err = err
r.Error = err.Error()
} else {
r.Status = Paused
r.Err = nil
r.Error = ""
}
}
func (r Request) Delete() error {
var err, ret error
if r.Response != nil {
err = r.Response.Cancel()
if err != nil && err != context.Canceled {
ret = err
}
}
_ = os.Remove(r.TempPath)
err = os.Remove(r.FilePath)
if err != nil && err != os.ErrNotExist {
ret = err
}
return ret
}
func newCookieJar() http.CookieJar {
c, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
return c
}
func newDownloader() *Downloader {
return &Downloader{
Jar: DefaultCookieJar,
Grab: DefaultGrabClient,
}
}
func (d *Downloader) Start(network, address string) {
var (
listener net.Listener
@ -183,7 +312,6 @@ func (d *Downloader) Start(network, address string) {
ReadTimeout: 2 * time.Minute,
WriteTimeout: 2 * time.Minute,
}
}
if d.DataDir == "" {
@ -209,26 +337,17 @@ func (d *Downloader) Start(network, address string) {
if err != nil {
panic(err)
}
log.Println("adding /add handler")
// mux.HandleFunc("/", d.UI)
log.Println("adding http handlers")
d.initStatus(mux)
mux.HandleFunc("/add", d.restAddDownload)
mux.HandleFunc("/queue", d.restQueueStatus)
mux.HandleFunc("/history", d.restHistoryStatus)
mux.HandleFunc("/start", d.restStartDownload)
mux.HandleFunc("/queued", d.restStatus(true))
mux.HandleFunc("/completed", d.restStatus(false))
mux.HandleFunc("/set", d.restSetDownloadStatus)
mux.HandleFunc("/delete", d.restDelete)
mux.Handle("/get/", http.StripPrefix("/get/", http.FileServer(http.Dir(d.CompleteDir))))
log.Println("starting main go routine")
d.Grab.HTTPClient = &http.Client{
Jar: d.Jar,
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
}
go d.download()
@ -236,64 +355,117 @@ func (d *Downloader) Start(network, address string) {
_ = d.Server.Serve(listener)
}
func (d *Downloader) restStartDownload(w http.ResponseWriter, r *http.Request) {
func httpMethodNotAllowed(w http.ResponseWriter, r *http.Request, method string) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", method)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintf(w, "HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
log.Printf("HTTP Error 405 – Method Not Allowed\nOnly %s method is allowed\n", method)
}
func (d *Downloader) restDelete(w http.ResponseWriter, r *http.Request) {
var (
err error
index struct {
index int
index []struct {
Index int
History bool
}
ret []Request
)
if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodPost)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
if r.Method != http.MethodDelete {
httpMethodNotAllowed(w, r, http.MethodDelete)
return
}
err = json.NewDecoder(r.Body).Decode(index)
err = json.NewDecoder(r.Body).Decode(&index)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if index.index >= d.Downloads.Len() || index.index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", index.index, d.Downloads.Len()), http.StatusBadRequest)
return
sort.Slice(index, func(i, j int) bool {
return index[i].Index < index[j].Index
})
for x, i := range index {
if i.History {
if i.Index >= d.History.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
ret = append(ret, *d.History.Pop(i.Index - x))
ret[len(ret)-1].Delete()
} else {
if i.Index >= d.Downloads.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
ret = append(ret, *d.Downloads.Pop(i.Index - x))
ret[len(ret)-1].Delete()
}
}
d.startDownload(index.index)
}
d.syncDownloads()
func (d *Downloader) restHistoryStatus(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodGet)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
v, err := jettison.MarshalOpts(ret, jettison.DenyList([]string{"cookies"}))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
j := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
j.Encode(d.History.Queue)
w.Write(v)
}
func (d *Downloader) restQueueStatus(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodGet)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly GET method is allowed")
func (d *Downloader) restSetDownloadStatus(w http.ResponseWriter, r *http.Request) {
var (
err error
index []struct {
Index int
Status Status
Priority Priority
}
)
if r.Method != http.MethodPost {
httpMethodNotAllowed(w, r, http.MethodPost)
return
}
j := json.NewEncoder(w)
w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
j.Encode(d.Downloads.Queue)
err = json.NewDecoder(r.Body).Decode(&index)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
for _, i := range index {
if i.Index >= d.Downloads.Len() || i.Index < 0 {
http.Error(w, fmt.Sprintf("slice index out of bounds. index: %d length of slice: %d", i.Index, d.Downloads.Len()), http.StatusBadRequest)
return
}
r := d.Downloads.Queue[i.Index]
r.Priority = i.Priority
r.Status = i.Status
}
d.syncDownloads()
}
func (d *Downloader) restStatus(q bool) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
var queue = &d.History
if q {
d.Downloads.updateStatus()
queue = &d.Downloads
}
if r.Method != http.MethodGet {
httpMethodNotAllowed(w, r, http.MethodGet)
return
}
queue.updateStatus()
v, err := jettison.MarshalOpts(queue.Queue, jettison.DenyList([]string{"cookies"}))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Add("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write(v)
}
}
func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
@ -302,12 +474,7 @@ func (d *Downloader) restAddDownload(w http.ResponseWriter, r *http.Request) {
err error
)
if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Add("Allow", http.MethodPost)
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(w, "HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
log.Println("HTTP Error 405 – Method Not Allowed\nOnly POST method is allowed")
httpMethodNotAllowed(w, r, http.MethodPost)
return
}
// TODO fail only on individual requests
@ -334,8 +501,10 @@ func (d Downloader) getNameFromHEAD(r Request) string {
re *http.Response
p map[string]string
)
r.insertCookies()
ht := &http.Client{
Jar: d.Jar,
Jar: r.Jar,
Timeout: 30 * time.Second,
Transport: &http.Transport{
Dial: (&net.Dialer{
@ -464,14 +633,6 @@ func (d Downloader) FindRequest(u string) *Request {
func (d *Downloader) addRequest(r *Request) {
log.Println("adding download for", r.URL)
req := d.FindRequest(r.URL)
u, _ := url.Parse(r.URL)
for i, v := range r.Cookies {
d.Jar.SetCookies(&url.URL{
Scheme: u.Scheme,
Path: v.Path,
Host: v.Domain,
}, []*http.Cookie{&r.Cookies[i]})
}
d.getFilename(r)
if req != nil { // url alread added
@ -502,20 +663,37 @@ func (d *Downloader) startDownload(i int) {
err error
)
r = d.Downloads.Queue[i]
r.insertCookies()
d.getTempFilename(r)
log.Println("starting download for", r.URL, "to", r.TempPath)
// d.Downloads.Queue = append(d.Downloads.Queue, r)
if r.Response == nil || r.Response.Err() != nil {
req, err = grab.NewRequest(r.TempPath, r.URL)
if err != nil {
r.Status = Error
r.Error = err
r.setError(err)
return
}
}
r.Status = Downloading
r.Response = d.Grab.Do(req)
if r.grab == nil {
r.grab = &grab.Client{
HTTPClient: &http.Client{
Jar: r.Jar,
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
},
}
}
r.Response = r.grab.Do(req)
go func(r *Request) {
log.Println("wait for download")
log.Println(r.Response.IsComplete())
@ -542,12 +720,43 @@ func (d *Downloader) syncDownloads() {
return
}
sort.Stable(d.Downloads)
var downloadsMaxed bool
// Start new downloads
for i, req := range d.Downloads.Queue {
if d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
if req.Status == Queued {
switch req.Status {
case Queued:
if !downloadsMaxed && d.MaxActiveDownloads >= len(d.getRunningDownloads()) {
d.startDownload(i)
}
case Stopped:
if req.Response != nil {
var err = req.Response.Cancel()
if err != nil && err != context.Canceled {
req.setError(err)
}
req.Response = nil
}
case Downloading:
if req.Response == nil {
d.startDownload(i)
}
case Error:
if req.Response != nil && req.Err == nil {
var err = req.Response.Err()
var err2 = req.Response.Cancel()
if err != nil && err2 != context.Canceled {
err = err2
}
req.setError(err)
}
case Canceled:
if req.Response != nil {
err := req.Response.Cancel()
if err != nil && err != context.Canceled {
req.setError(err)
}
req.Response = nil
}
}
}
@ -558,6 +767,7 @@ func (d *Downloader) syncDownloads() {
i--
}
}
d.Downloads.updateStatus()
}
func (d *Downloader) requestCompleted(r *Request) {
@ -572,9 +782,8 @@ func (d *Downloader) requestCompleted(r *Request) {
}
d.History.Queue = append(d.History.Queue, r)
} else {
r.Status = Error
r.Error = r.Response.Err()
log.Println("fucking error:", r.Error)
r.setError(r.Response.Err())
log.Println("fucking error:", r.Err)
}
}
@ -585,17 +794,93 @@ func (d *Downloader) download() {
d.syncDownloads()
case r := <-d.NewRequest:
r.Progress = "" // not allowed when adding
d.addRequest(&r)
if d.OnAdd != nil {
d.OnAdd(r)
d.OnAdd(d, r)
}
case r := <-d.requestDone:
log.Println("finishing request for", r.URL)
d.requestCompleted(r)
if d.OnComplete != nil {
d.OnComplete(*r)
d.OnComplete(d, *r)
}
}
}
}
func (r *Request) insertCookies() {
if r.Jar == nil {
r.Jar = newCookieJar()
}
u, _ := url.Parse(r.URL)
for i, v := range r.Cookies {
r.Jar.SetCookies(&url.URL{
Scheme: u.Scheme,
Path: v.Path,
Host: v.Domain,
}, []*http.Cookie{&r.Cookies[i]})
}
}
func (d *Downloader) initStatus(mux *http.ServeMux) {
commonTmpls := template.New("root").Funcs(map[string]interface{}{
"shortenSHA256": func(hash string) string {
if len(hash) > 10 {
return hash[:10]
}
return hash
},
})
commonTmpls = template.Must(commonTmpls.New("header").Parse(bundled.Asset("header.tmpl")))
commonTmpls = template.Must(commonTmpls.New("footer").Parse(bundled.Asset("footer.tmpl")))
queueTmpl := template.Must(template.Must(commonTmpls.Clone()).New("queueTmpl").Parse(bundled.Asset("queue.tmpl")))
historyTmpl := template.Must(template.Must(commonTmpls.Clone()).New("historyTmpl").Parse(bundled.Asset("history.tmpl")))
for _, fn := range []string{
"favicon.ico",
"bootstrap.min.js",
"bootstrap.min.css",
"bootstrap-table.min.js",
"bootstrap-table.min.css",
"popper.min.js",
"jquery-3.2.1.slim.min.js",
"downloads.js",
} {
mux.Handle("/"+fn, bundled.HTTPHandlerFunc(fn))
}
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
d.Downloads.updateStatus()
if err := queueTmpl.Execute(&buf, struct {
Queue []*Request
History []*Request
Hostname string
}{
Queue: d.Downloads.Queue,
History: d.History.Queue,
Hostname: "gloader",
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
io.Copy(w, &buf)
})
mux.HandleFunc("/history", func(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
d.Downloads.updateStatus()
if err := historyTmpl.Execute(&buf, struct {
Queue []*Request
History []*Request
Hostname string
}{
Queue: d.Downloads.Queue,
History: d.History.Queue,
Hostname: "gloader",
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
io.Copy(w, &buf)
})
}

1
go.mod
View File

@ -8,5 +8,6 @@ require (
github.com/cavaliercoder/grab v2.0.0+incompatible
github.com/lordwelch/pathvalidate v0.0.0-20201012043703-54efa7ea1308
github.com/u-root/u-root v7.0.0+incompatible
github.com/wI2L/jettison v0.7.1
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
)

187
goembed.go Normal file
View File

@ -0,0 +1,187 @@
// +build ignore
// goembed generates a Go source file from an input file.
package main
import (
"bufio"
"bytes"
"compress/gzip"
"flag"
"fmt"
"io"
"log"
"os"
"text/template"
"unicode/utf8"
)
var (
packageFlag = flag.String("package", "", "Go package name")
varFlag = flag.String("var", "", "Go var name")
gzipFlag = flag.Bool("gzip", false, "Whether to gzip contents")
)
func main() {
flag.Parse()
fmt.Printf("package %s\n\n", *packageFlag)
if *gzipFlag {
err := gzipPrologue.Execute(os.Stdout, map[string]interface{}{
"Args": flag.Args(),
"VarName": *varFlag,
})
if err != nil {
log.Fatal(err)
}
}
if flag.NArg() > 0 {
fmt.Println("// Table of contents")
fmt.Printf("var %v = map[string][]byte{\n", *varFlag)
for i, filename := range flag.Args() {
fmt.Printf("\t%q: %s_%d,\n", filename, *varFlag, i)
}
fmt.Println("}")
// Using a separate variable for each []byte, instead of
// combining them into a single map literal, enables a storage
// optimization: the compiler places the data directly in the
// program's noptrdata section instead of the heap.
for i, filename := range flag.Args() {
if err := oneVar(fmt.Sprintf("%s_%d", *varFlag, i), filename); err != nil {
log.Fatal(err)
}
}
} else {
if err := oneVarReader(*varFlag, os.Stdin); err != nil {
log.Fatal(err)
}
}
}
func oneVar(varName, filename string) error {
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
return oneVarReader(varName, f)
}
func oneVarReader(varName string, r io.Reader) error {
// Generate []byte(<big string constant>) instead of []byte{<list of byte values>}.
// The latter causes a memory explosion in the compiler (60 MB of input chews over 9 GB RAM).
// Doing a string conversion avoids some of that, but incurs a slight startup cost.
if !*gzipFlag {
fmt.Printf(`var %s = []byte("`, varName)
} else {
var buf bytes.Buffer
gzw, _ := gzip.NewWriterLevel(&buf, gzip.BestCompression)
if _, err := io.Copy(gzw, r); err != nil {
return err
}
if err := gzw.Close(); err != nil {
return err
}
fmt.Printf("var %s []byte // set in init\n\n", varName)
fmt.Printf(`var %s_gzip = []byte("`, varName)
r = &buf
}
bufw := bufio.NewWriter(os.Stdout)
if _, err := io.Copy(&writer{w: bufw}, r); err != nil {
return err
}
if err := bufw.Flush(); err != nil {
return err
}
fmt.Println(`")`)
return nil
}
type writer struct {
w io.Writer
}
func (w *writer) Write(data []byte) (n int, err error) {
n = len(data)
for err == nil && len(data) > 0 {
// https://golang.org/ref/spec#String_literals: "Within the quotes, any
// character may appear except newline and unescaped double quote. The
// text between the quotes forms the value of the literal, with backslash
// escapes interpreted as they are in rune literals […]."
switch b := data[0]; b {
case '\\':
_, err = w.w.Write([]byte(`\\`))
case '"':
_, err = w.w.Write([]byte(`\"`))
case '\n':
_, err = w.w.Write([]byte(`\n`))
case '\x00':
// https://golang.org/ref/spec#Source_code_representation: "Implementation
// restriction: For compatibility with other tools, a compiler may
// disallow the NUL character (U+0000) in the source text."
_, err = w.w.Write([]byte(`\x00`))
default:
// https://golang.org/ref/spec#Source_code_representation: "Implementation
// restriction: […] A byte order mark may be disallowed anywhere else in
// the source."
const byteOrderMark = '\uFEFF'
if r, size := utf8.DecodeRune(data); r != utf8.RuneError && r != byteOrderMark {
_, err = w.w.Write(data[:size])
data = data[size:]
continue
}
_, err = fmt.Fprintf(w.w, `\x%02x`, b)
}
data = data[1:]
}
return n - len(data), err
}
var gzipPrologue = template.Must(template.New("").Parse(`
import (
"bytes"
"compress/gzip"
"io/ioutil"
)
func init() {
var (
r *gzip.Reader
err error
)
{{ if gt (len .Args) 0 }}
{{ range $idx, $var := .Args }}
{{ $n := printf "%s_%d" $.VarName $idx }}
r, err = gzip.NewReader(bytes.NewReader({{ $n }}_gzip))
if err != nil {
panic(err)
}
{{ $n }}, err = ioutil.ReadAll(r)
{{ $.VarName }}["{{ $var }}"] = {{ $n }}
r.Close()
if err != nil {
panic(err)
}
{{ end }}
{{ else }}
r, err = gzip.NewReader(bytes.NewReader({{ .VarName }}_gzip))
if err != nil {
panic(err)
}
{{ .VarName }}, err = ioutil.ReadAll(r)
r.Close()
if err != nil {
panic(err)
}
{{ end }}
}
`))

3
grab/client.go
View File

@ -267,6 +267,7 @@ func (c *Client) validateLocal(resp *Response) stateFunc {
if expectedSize >= 0 && expectedSize < resp.fi.Size() {
// remote size is known, is smaller than local size and we want to resume
fmt.Fprintln(os.Stderr, "validate\n")
resp.err = ErrBadLength
return c.closeResponse
}
@ -395,6 +396,7 @@ func (c *Client) readResponse(resp *Response) stateFunc {
// remote size is known
resp.sizeUnsafe += resp.bytesResumed
if resp.Request.Size > 0 && resp.Request.Size != resp.sizeUnsafe {
fmt.Fprintln(os.Stderr, "response\n")
resp.err = ErrBadLength
return c.closeResponse
}
@ -527,6 +529,7 @@ func (c *Client) copyFile(resp *Response) stateFunc {
discoveredSize := resp.bytesResumed + bytesCopied
atomic.StoreInt64(&resp.sizeUnsafe, discoveredSize)
if resp.Request.Size > 0 && resp.Request.Size != discoveredSize {
fmt.Fprintln(os.Stderr, "file\n")
resp.err = ErrBadLength
return c.closeResponse
}

66
main.go
View File

@ -32,40 +32,42 @@ func main() {
log.Println(err)
os.Exit(1)
}
d := newDownloader()
loadQueue(d)
save := func(r Request) {
var (
content []byte
err error
)
content, err = json.Marshal(d.History.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "history.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
content, err = json.Marshal(d.Downloads.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "queue.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
d := &Downloader{
OnAdd: save,
OnComplete: save,
DataDir: filepath.Join(gloaderHome, "data"),
}
d.OnAdd = save
d.OnComplete = save
d.DataDir = filepath.Join(gloaderHome, "data")
loadQueue(d)
d.Start("tcp", ":8844")
}
func save(d *Downloader, r Request) {
var (
content []byte
err error
)
content, err = json.Marshal(d.History.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "history.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
content, err = json.Marshal(d.Downloads.Queue)
if err != nil {
log.Println(err)
return
}
err = ioutil.WriteFile(filepath.Join(gloaderHome, "queue.json"), content, 0o666)
if err != nil {
log.Println(err)
return
}
}
func loadQueue(d *Downloader) {
var (
f io.ReadCloser
@ -142,6 +144,10 @@ func mount() error {
if err != nil {
return fmt.Errorf("error mounting datadir: %w", err)
}
err = syscall.Mount(dev.Path, dataDir, "ext4", syscall.MS_SHARED|syscall.MS_REMOUNT, "")
if err != nil {
return fmt.Errorf("error remounting datadir: %w", err)
}
return nil
}
return fmt.Errorf("error mounting datadir: %w", err)