Use `got` for http requests throughout the codebase about blurts-server

[pdehaan]

Counter argument: Let's switch to the slimmer Promise-based got by Sindre:

"Created because request is bloated (several megabytes!)."

[nhnt11]

Updated the issue title, thanks for the suggestion! I haven't checked out got yet but I trust your judgement.

[pdehaan]

Rough stab at get_hashsets.js:

const AppConstants = require("./app-constants").init();

const fs = require("fs");
const path = require("path");
const got = require("got");
const {name, version} = require("./package.json");

const BREACH_HASHSET_DIR = "breach_hashsets";
const HIBP_AUTH = `Bearer ${AppConstants.HIBP_API_TOKEN}`;
const HIBP_DEFAULT_HEADERS = {"User-Agent": `${name}/${version}`};


async function getBreaches() {
  const url = `${AppConstants.HIBP_API_ROOT}/breaches`;
  const headers = {...HIBP_DEFAULT_HEADERS};
  try {
    const {body} = await got.get(url, {json: true, headers});
    if (!fs.existsSync(BREACH_HASHSET_DIR)) {
      fs.mkdirSync(BREACH_HASHSET_DIR);
    }
    return body.map(async (breach) => await getBreachHashset(breach));
  } catch(err) {
    console.error(err);
  }
}

async function getBreachHashset({IsActive, IsVerified, DataClasses, Name}) {
  if (IsActive && IsVerified && DataClasses.includes("Email addresses")) {
    const BREACH_HASHSET_ZIP = path.join(BREACH_HASHSET_DIR, `${Name}.zip`);
    const url = `${AppConstants.HIBP_API_ROOT}/enterprisesubscriber/hashset/${Name}`;
    const headers = {
      ...HIBP_DEFAULT_HEADERS,
      Authorization: HIBP_AUTH
    };

    try {
      const zipStream = fs.createWriteStream(BREACH_HASHSET_ZIP);
      console.log(`Active, verified breach with email addresses: ${Name}`);
      console.log(`Fetching ${url}...`);
      return await got.stream(url, {headers}).pipe(zipStream);
    } catch(err) {
      console.log(`${err.message} -- ${url}`);
    }
  }
}

getBreaches();

... And I think we can maybe just use the following for the routes/oauth.js code:

return got.get(FxAOAuthUtils.profileUri, {json: true, headers: `Bearer ${user.accessToken}`})
  .then(({body}) => {
    const email = body.email;
    gEmails.add(email);
    res.send(`Registered ${email} for breach alerts. You may now close this window/tab.`);
  })
  .catch(err => res.send(err));

(and possibly even destructure body.email a bit more succinctly to shave an extra line off that snippet)

.then(({body: email}) => {

old:

blurts-server/routes/oauth.js

Lines 48 to 63 in 2900cc2

	return popsicle.get({
	method: "get",
	url: FxAOAuthUtils.profileUri,
	body: "",
	headers: {
	Authorization: `Bearer ${user.accessToken}`,
	},
	}).then((data) => {
	const email = JSON.parse(data.body).email;
	gEmails.add(email);
	res.send(`Registered ${email} for breach alerts. You may now close this window/tab.`);
	});
	})
	.catch((err) => {
	res.send(err);
	});

[groovecoder]

I briefly tried got.stream for the hashsets, but it seems to fail harder on 404's and kill the process? I didn't dig into debugging it.

[pdehaan]

@groovecoder This may get us closer...

Using got.stream() here to download the ZIP file of master branch (easier to fiddle w/ 404s by tweaking the URL):

const fs = require("fs");
const path = require("path");
const got = require("got");

const SRC_FILE = "https://github.com/mozilla/blurts-server/archive/master.zip";
const DEST_FILE = path.join(__dirname, "tmp", "master.zip");

async function request(inUrl, outPath) {
  return await got.stream.get(inUrl)
    .on('error', (error, body, {requestUrl, statusCode}) => {
      console.error(`[${statusCode}] ${error.message}: ${requestUrl}`);
    })
    .on('response', ({requestUrl, statusCode}) => {
      console.log(`[${statusCode}] Downloaded: ${requestUrl}`);
    })
    .pipe(fs.createWriteStream(outPath));
}

request(SRC_FILE, DEST_FILE);

> node get-master
# [200] Downloaded: https://github.com/mozilla/blurts-server/archive/master.zip

> node get-master
# [404] Response code 404 (Not Found): https://github.com/mozilla/blurts-server/archive/masterz.zip