Send referrer with prerolls

See #89

lib/browser.ts

@@ -40,9 +40,61 @@ export interface RawPort {
   postMessage: (message: any) => void;
 }
 
+interface WebRequestFilter {
+  urls?: string[];
+}
+
+interface WebRequestListener {
+  addListener(
+    callback: Function,
+    filter: WebRequestFilter,
+    extraInfoSpec: string[]
+    ): void;
+  removeListener(callback: Function): void;
+}
+
+type Header = {name: string; value: string};
+
+export interface DownloadOptions {
+  conflictAction: string;
+  filename: string;
+  saveAs: boolean;
+  url: string;
+  method?: string;
+  body?: string;
+  incognito?: boolean;
+  headers: Header[];
+}
+
+export interface DownloadsQuery {
+  id?: number;
+}
+
+interface Downloads {
+  download(download: DownloadOptions): Promise<number>;
+  open(manId: number): Promise<void>;
+  show(manId: number): Promise<void>;
+  pause(manId: number): Promise<void>;
+  resume(manId: number): Promise<void>;
+  cancel(manId: number): Promise<void>;
+  erase(query: DownloadsQuery): Promise<void>;
+  search(query: DownloadsQuery): Promise<any[]>;
+  getFileIcon(id: number, options?: any): Promise<string>;
+  setShelfEnabled(state: boolean): void;
+  onCreated: ExtensionListener;
+  onChanged: ExtensionListener;
+  onErased: ExtensionListener;
+}
+
+interface WebRequest {
+  onBeforeSendHeaders: WebRequestListener;
+  onSendHeaders: WebRequestListener;
+  onHeadersReceived: WebRequestListener;
+}
+
 export const {browserAction} = polyfill;
 export const {contextMenus} = polyfill;
-export const {downloads} = polyfill;
+export const {downloads}: {downloads: Downloads} = polyfill;
 export const {extension} = polyfill;
 export const {history} = polyfill;
 export const {menus} = polyfill;
@@ -52,7 +104,7 @@ export const {sessions} = polyfill;
 export const {storage} = polyfill;
 export const {tabs} = polyfill;
 export const {webNavigation} = polyfill;
-export const {webRequest} = polyfill;
+export const {webRequest}: {webRequest: WebRequest} = polyfill;
 export const {windows} = polyfill;
 
 export const CHROME = navigator.appVersion.includes("Chrome/");

lib/manager/download.ts

@@ -1,7 +1,8 @@
 "use strict";
 // License: MIT
 
-import { CHROME, downloads } from "../browser";
+// eslint-disable-next-line no-unused-vars
+import { CHROME, downloads, DownloadOptions } from "../browser";
 import { Prefs } from "../prefs";
 import { PromiseSerializer } from "../pserializer";
 import { filterInSitu, parsePath } from "../util";
@@ -22,18 +23,6 @@ import {
 } from "./state";
 import { Preroller } from "./preroller";
 
-type Header = {name: string; value: string};
-interface Options {
-  conflictAction: string;
-  filename: string;
-  saveAs: boolean;
-  url: string;
-  method?: string;
-  body?: string;
-  incognito?: boolean;
-  headers: Header[];
-}
-
 export class Download extends BaseDownload {
   public manager: Manager;
 
@@ -120,7 +109,7 @@ export class Download extends BaseDownload {
           return;
         }
       }
-      const options: Options = {
+      const options: DownloadOptions = {
         conflictAction: await Prefs.get("conflict-action"),
         filename: this.dest.full,
         saveAs: false,
@@ -140,6 +129,12 @@ export class Download extends BaseDownload {
           value: this.referrer
         });
       }
+      else if (CHROME) {
+        options.headers.push({
+          name: "X-DTA-ID",
+          value: this.sessionId.toString(),
+        });
+      }
       if (this.manId) {
         this.manager.removeManId(this.manId);
       }

lib/manager/man.ts

@@ -16,8 +16,9 @@ import { Download } from "./download";
 import { ManagerPort } from "./port";
 import { Scheduler } from "./scheduler";
 import { Limits } from "./limits";
-import { downloads, runtime } from "../browser";
+import { downloads, runtime, webRequest, CHROME } from "../browser";
 
+const US = runtime.getURL("");
 
 const AUTOSAVE_TIMEOUT = 2000;
 const DIRTY_TIMEOUT = 100;
@@ -83,6 +84,14 @@ export class Manager extends EventEmitter {
     Limits.on("changed", () => {
       this.resetScheduler();
     });
+
+    if (CHROME) {
+      webRequest.onBeforeSendHeaders.addListener(
+        this.stuffReferrer.bind(this),
+        {urls: ["<all_urls>"]},
+        ["blocking", "requestHeaders", "extraHeaders"]
+      );
+    }
   }
 
   async init() {
@@ -384,6 +393,31 @@ export class Manager extends EventEmitter {
   getMsgItems() {
     return this.items.map(e => e.toMsg());
   }
+
+  stuffReferrer(details: any): any {
+    if (details.tabId > 0 && !US.startsWith(details.initiator)) {
+      return undefined;
+    }
+    const sidx = details.requestHeaders.findIndex(
+      (e: any) => e.name.toLowerCase() === "x-dta-id");
+    if (sidx < 0) {
+      return undefined;
+    }
+    const sid = parseInt(details.requestHeaders[sidx].value, 10);
+    details.requestHeaders.splice(sidx, 1);
+    const item = this.sids.get(sid);
+    if (!item) {
+      return undefined;
+    }
+    details.requestHeaders.push({
+      name: "Referer",
+      value: (item.uReferrer || item.uURL).toString()
+    });
+    const rv: any = {
+      requestHeaders: details.requestHeaders
+    };
+    return rv;
+  }
 }
 
 let inited: Promise<Manager>;

lib/manager/preroller.ts

@@ -79,7 +79,7 @@ export class Preroller {
   private async prerollFirefox() {
     const controller = new AbortController();
     const {signal} = controller;
-    const {uURL} = this.download;
+    const {uURL, uReferrer} = this.download;
     const res = await fetch(uURL.toString(), {
       method: "GET",
       headers: new Headers({
@@ -87,6 +87,7 @@ export class Preroller {
       }),
       mode: "same-origin",
       signal,
+      referrer: (uReferrer || uURL).toString(),
     });
     if (res.body) {
       res.body.cancel();
@@ -98,7 +99,7 @@ export class Preroller {
 
   private async prerollChrome() {
     let rid = "";
-    const {uURL} = this.download;
+    const {uURL, uReferrer} = this.download;
     const rurl = uURL.toString();
     let listener: any;
     const wr = new Promise<any[]>(resolve => {
@@ -132,9 +133,11 @@ export class Preroller {
     const res = await fetch(rurl, {
       method: "GET",
       headers: new Headers({
-        Range: "bytes=0-1",
+        "Range": "bytes=0-1",
+        "X-DTA-ID": this.download.sessionId.toString(),
       }),
       signal,
+      referrer: (uReferrer || uURL).toString(),
     });
     if (res.body) {
       res.body.cancel();

manifest.json

@@ -34,7 +34,8 @@
     "storage",
     "tabs",
     "webNavigation",
-    "webRequest"
+    "webRequest",
+    "webRequestBlocking"
   ],
 
   "background": {

util/build.py

@@ -26,7 +26,7 @@ UNCOMPRESSABLE = set((".png", ".jpg", ".zip", ".woff2"))
 LICENSED = set((".css", ".html", ".js", "*.ts"))
 IGNORED = set((".DS_Store", "Thumbs.db"))
 
-PERM_IGNORED_FX = set(("downloads.shelf", "webRequest"))
+PERM_IGNORED_FX = set(("downloads.shelf", "webRequest", "webRequestBlocking"))
 PERM_IGNORED_CHROME = set(("menus", "sessions"))
 
 SCRIPTS = [