feat: add and use visit queue

server/controllers/authController.ts

@@ -144,35 +144,29 @@ export const signup: Handler = async (req, res) => {
 
   const newUser = await createUser(email, password, user);
 
-  try {
-
-
-    const mail = await transporter.sendMail({
-      from: process.env.MAIL_FROM || process.env.MAIL_USER,
-      to: newUser.email,
-      subject: "Verify your account",
-      text: verifyMailText.replace(
-        /{{verification}}/gim,
-        newUser.verification_token
-      ),
-      html: verifyEmailTemplate.replace(
-        /{{verification}}/gim,
-        newUser.verification_token
-      )
-    });
-
-    if (mail.accepted.length) {
-      return res
-        .status(201)
-        .json({ email, message: "Verification email has been sent." });
-    }
+  const mail = await transporter.sendMail({
+    from: process.env.MAIL_FROM || process.env.MAIL_USER,
+    to: newUser.email,
+    subject: "Verify your account",
+    text: verifyMailText.replace(
+      /{{verification}}/gim,
+      newUser.verification_token
+    ),
+    html: verifyEmailTemplate.replace(
+      /{{verification}}/gim,
+      newUser.verification_token
+    )
+  });
 
+  if (mail.accepted.length) {
     return res
-      .status(400)
-      .json({ error: "Couldn't send verification email. Try again." });
-  } catch (error) {
-    console.log({ error });
+      .status(201)
+      .json({ email, message: "Verification email has been sent." });
   }
+
+  return res
+    .status(400)
+    .json({ error: "Couldn't send verification email. Try again." });
 };
 
 export const login: Handler = (req, res) => {

server/controllers/linkController.ts

@@ -1,21 +1,17 @@
 import bcrypt from "bcryptjs";
 import dns from "dns";
 import { Handler } from "express";
-import geoip from "geoip-lite";
 import isbot from "isbot";
 import generate from "nanoid/generate";
 import ua from "universal-analytics";
 import URL from "url";
 import urlRegex from "url-regex";
-import useragent from "useragent";
 import { promisify } from "util";
 import { deleteDomain, getDomain, setDomain } from "../db/domain";
 import { addIP } from "../db/ip";
 import {
-  addLinkCount,
   banLink,
   createShortLink,
-  createVisit,
   deleteLink,
   findLink,
   getLinks,
@@ -24,12 +20,7 @@ import {
 } from "../db/link";
 import transporter from "../mail/mail";
 import * as redis from "../redis";
-import {
-  addProtocol,
-  generateShortLink,
-  getStatsCacheTime,
-  getStatsLimit
-} from "../utils";
+import { addProtocol, generateShortLink, getStatsCacheTime } from "../utils";
 import {
   checkBannedDomain,
   checkBannedHost,
@@ -38,6 +29,7 @@ import {
   preservedUrls,
   urlCountsCheck
 } from "./validateBodyController";
+import { visitQueue } from "../queues";
 
 const dnsLookup = promisify(dns.lookup);
 
@@ -119,29 +111,17 @@ export const shortener: Handler = async (req, res) => {
   }
 };
 
-const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"];
-const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"];
-const filterInBrowser = agent => item =>
-  agent.family.toLowerCase().includes(item.toLocaleLowerCase());
-const filterInOs = agent => item =>
-  agent.os.family.toLowerCase().includes(item.toLocaleLowerCase());
-
 export const goToLink: Handler = async (req, res, next) => {
   const { host } = req.headers;
   const reqestedId = req.params.id || req.body.id;
   const address = reqestedId.replace("+", "");
   const customDomain = host !== process.env.DEFAULT_DOMAIN && host;
-  // TODO: Extract parsing into their own function
-  const agent = useragent.parse(req.headers["user-agent"]);
-  const [browser = "Other"] = browsersList.filter(filterInBrowser(agent));
-  const [os = "Other"] = osList.filter(filterInOs(agent));
-  const referrer =
-    req.header("Referer") && URL.parse(req.header("Referer")).hostname;
-  const location = geoip.lookup(req.realIP);
-  const country = location && location.country;
   const isBot = isbot(req.headers["user-agent"]);
 
-  const domain = await (customDomain && getDomain({ address: customDomain }));
+  let domain;
+  if (customDomain) {
+    domain = await getDomain({ address: customDomain });
+  }
 
   const link = await findLink({ address, domain_id: domain && domain.id });
 
@@ -176,29 +156,24 @@ export const goToLink: Handler = async (req, res, next) => {
       return res.status(401).json({ error: "Password is not correct" });
     }
     if (link.user_id && !isBot) {
-      addLinkCount(link.id);
-      createVisit({
-        browser: browser.toLowerCase(),
-        country: country || "Unknown",
-        domain: customDomain,
-        id: link.id,
-        os: os.toLowerCase().replace(/\s/gi, ""),
-        referrer: referrer.replace(/\./gi, "[dot]") || "Direct",
-        limit: getStatsLimit()
+      visitQueue.add({
+        headers: req.headers,
+        realIP: req.realIP,
+        referrer: req.get("Referrer"),
+        link,
+        customDomain
       });
     }
     return res.status(200).json({ target: link.target });
   }
+
   if (link.user_id && !isBot) {
-    addLinkCount(link.id);
-    createVisit({
-      browser: browser.toLowerCase(),
-      country: (country && country.toLocaleLowerCase()) || "unknown",
-      domain: customDomain,
-      id: link.id,
-      os: os.toLowerCase().replace(/\s/gi, ""),
-      referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "direct",
-      limit: getStatsLimit()
+    visitQueue.add({
+      headers: req.headers,
+      realIP: req.realIP,
+      referrer: req.get("Referrer"),
+      link,
+      customDomain
     });
   }
 

server/db/link.ts

@@ -399,7 +399,6 @@ export const getStats = async (link: Link, domain: Domain) => {
       set(new Date(), { date: 1 }),
       set(new Date(visit.created_at), { date: 1 })
     );
-    console.log(diff);
     const index = stats.allTime.views.length - diff - 1;
     const view = stats.allTime.views[index];
     stats.allTime.stats = {

server/queues.ts

@@ -0,0 +1,48 @@
+import Queue from "bull";
+import useragent from "useragent";
+import geoip from "geoip-lite";
+import URL from "url";
+
+import { createVisit, addLinkCount } from "./db/link";
+import { getStatsLimit } from "./utils";
+
+const redis = {
+  port: Number(process.env.REDIS_PORT) || 6379,
+  host: process.env.REDIS_HOST || "127.0.0.1",
+  ...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD })
+};
+
+export const visitQueue = new Queue("visit", { redis });
+
+const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"];
+const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"];
+const filterInBrowser = agent => item =>
+  agent.family.toLowerCase().includes(item.toLocaleLowerCase());
+const filterInOs = agent => item =>
+  agent.os.family.toLowerCase().includes(item.toLocaleLowerCase());
+
+visitQueue.process(({ data }) => {
+  const agent = useragent.parse(data.headers["user-agent"]);
+  const [browser = "Other"] = browsersList.filter(filterInBrowser(agent));
+  const [os = "Other"] = osList.filter(filterInOs(agent));
+  const referrer = data.referrer && URL.parse(data.referrer).hostname;
+  const location = geoip.lookup(data.realIP);
+  const country = location && location.country;
+
+  return Promise.all([
+    addLinkCount(data.link.id),
+    createVisit({
+      browser: browser.toLowerCase(),
+      country: country || "Unknown",
+      domain: data.customDomain,
+      id: data.link.id,
+      os: os.toLowerCase().replace(/\s/gi, ""),
+      referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "Direct",
+      limit: getStatsLimit()
+    })
+  ]);
+});
+
+visitQueue.on("active", (job, prom) => {
+  console.dir({ job, prom }, Infinity);
+});

server/server.ts

@@ -56,7 +56,6 @@ app.prepare().then(async () => {
   server.use(express.static("static"));
 
   server.use((error, req, res, next) => {
-    console.log({ error });
     res
       .status(500)
       .json({ error: "Sorry an error ocurred. Please try again later." });