fix handling dates across databases

server/cron.js

@@ -1,6 +1,7 @@
 const cron = require("node-cron");
 
 const query = require("./queries");
+const utils = require("./utils");
 const env = require("./env");
 
 if (env.NON_USER_COOLDOWN) {
@@ -9,6 +10,6 @@ if (env.NON_USER_COOLDOWN) {
   });
 }
 
-cron.schedule("*/15 * * * * *", function() {
-  query.link.batchRemove({ expire_in: ["<", new Date().toISOString()] }).catch();
+cron.schedule("*/1 * * * * *", function() {
+  query.link.batchRemove({ expire_in: ["<", utils.dateToUTC(new Date())] }).catch();
 });

server/handlers/auth.handler.js

@@ -87,12 +87,11 @@ async function cooldown(req, res, next) {
   
   const ip = await query.ip.find({
     ip: req.realIP.toLowerCase(),
-    created_at: [">", subMinutes(new Date(), cooldownConfig).toISOString()]
+    created_at: [">", utils.dateToUTC(subMinutes(new Date(), cooldownConfig))]
   });
   
   if (ip) {
-    const timeToWait =
-      cooldownConfig - differenceInMinutes(new Date(), new Date(ip.created_at));
+    const timeToWait = cooldownConfig - differenceInMinutes(new Date(), utils.parseDatetime(ip.created_at));
     throw new CustomError(
       `Non-logged in users are limited. Wait ${timeToWait} minutes or log in.`,
       400
@@ -142,7 +141,7 @@ async function verify(req, res, next) {
 
   const user = await query.user.find({
     verification_token: req.params.verificationToken,
-    verification_expires: [">", new Date().toISOString()]
+    verification_expires: [">", utils.dateToUTC(new Date())]
   });
 
   if (!user) return next();
@@ -223,7 +222,7 @@ async function resetPasswordRequest(req, res) {
     { email: req.body.email },
     {
       reset_password_token: uuid(),
-      reset_password_expires: addMinutes(new Date(), 30).toISOString()
+      reset_password_expires: utils.dateToUTC(addMinutes(new Date(), 30))
     }
   );
 
@@ -252,7 +251,7 @@ async function resetPassword(req, res, next) {
     const [user] = await query.user.update(
       {
         reset_password_token: resetPasswordToken,
-        reset_password_expires: [">", new Date().toISOString()]
+        reset_password_expires: [">", utils.dateToUTC(new Date())]
       },
       { reset_password_expires: null, reset_password_token: null }
     );
@@ -293,7 +292,7 @@ async function changeEmailRequest(req, res) {
     {
       change_email_address: email,
       change_email_token: uuid(),
-      change_email_expires: addMinutes(new Date(), 30).toISOString()
+      change_email_expires: utils.dateToUTC(addMinutes(new Date(), 30))
     }
   );
   
@@ -322,7 +321,7 @@ async function changeEmail(req, res, next) {
   if (changeEmailToken) {
     const foundUser = await query.user.find({
       change_email_token: changeEmailToken,
-      change_email_expires: [">", new Date().toISOString()]
+      change_email_expires: [">", utils.dateToUTC(new Date())]
     });
   
     if (!foundUser) return next();

server/handlers/links.handler.js

@@ -151,8 +151,8 @@ async function edit(req, res) {
       delete req.body[name];
       return;
     }
-    if (name === "expire_in")
-      if (differenceInSeconds(new Date(value), new Date(link.expire_in)) <= 60) 
+    if (name === "expire_in" && link.expire_in)
+      if (Math.abs(differenceInSeconds(utils.parseDatetime(value), utils.parseDatetime(link.expire_in))) < 60)
           return;
     if (name === "password")
       if (value && value.replace(/•/ig, "").length === 0) {

server/handlers/validators.handler.js

@@ -75,7 +75,7 @@ const createLink = [
     .customSanitizer(ms)
     .custom(value => value >= ms("1m"))
     .withMessage("Expire time should be more than 1 minute.")
-    .customSanitizer(value => addMilliseconds(new Date(), value).toISOString()),
+    .customSanitizer(value => utils.dateToUTC(addMilliseconds(new Date(), value))),
   body("domain")
     .optional({ nullable: true, checkFalsy: true })
     .customSanitizer(value => value === env.DEFAULT_DOMAIN ? null : value)
@@ -138,7 +138,7 @@ const editLink = [
     .customSanitizer(ms)
     .custom(value => value >= ms("1m"))
     .withMessage("Expire time should be more than 1 minute.")
-    .customSanitizer(value => addMilliseconds(new Date(), value).toISOString()),
+    .customSanitizer(value => utils.dateToUTC(addMilliseconds(new Date(), value))),
   body("description")
     .optional({ nullable: true, checkFalsy: true })
     .isString()
@@ -346,7 +346,7 @@ function cooldown(user) {
   if (!user?.cooldown) return;
 
   // If user has active cooldown then throw error
-  const hasCooldownNow = differenceInHours(new Date(), new Date(user.cooldown)) < 12;
+  const hasCooldownNow = differenceInHours(new Date(), utils.parseDatetime(user.cooldown)) < 12;
 
   if (hasCooldownNow) {
     throw new utils.CustomError("Cooldown because of a malware URL. Wait 12h");
@@ -391,7 +391,7 @@ async function malware(user, target) {
   if (user) {
     const [updatedUser] = await query.user.update(
       { id: user.id },
-      { cooldown: new Date().toISOString() },
+      { cooldown: utils.dateToUTC(new Date()) },
       { increments: ["malicious_attempts"] }
     );
 
@@ -412,7 +412,7 @@ async function linksCount(user) {
 
   const count = await query.link.total({
     user_id: user.id,
-    "links.created_at": [">", subDays(new Date(), 1).toISOString()]
+    "links.created_at": [">", utils.dateToUTC(subDays(new Date(), 1))]
   });
 
   if (count > env.USER_LIMIT_PER_DAY) {

server/knex.js

@@ -20,4 +20,6 @@ const db = knex({
   useNullAsDefault: true,
 });
 
+db.isSqlite3 = db.client.driverName === "sqlite3" || db.client.driverName === "better-sqlite3";
+
 module.exports = db;

server/queries/domain.queries.js

@@ -1,4 +1,5 @@
 const redis = require("../redis");
+const utils = require("../utils");
 const knex = require("../knex");
 
 async function find(match) {
@@ -43,7 +44,7 @@ async function add(params) {
   if (id) {
     await knex("domains").where("id", id).update({
       ...newDomain,
-      updated_at: params.updated_at || new Date().toISOString()
+      updated_at: params.updated_at || utils.dateToUTC(new Date())
     });
   } else {
     // Mysql and sqlite don't support returning but return the inserted id by default
@@ -62,7 +63,7 @@ async function add(params) {
 async function update(match, update) {
   await knex("domains")
     .where(match)
-    .update({ ...update, updated_at: new Date().toISOString() });
+    .update({ ...update, updated_at: utils.dateToUTC(new Date()) });
 
   const domains = await knex("domains").select("*").where(match);
 

server/queries/host.queries.js

@@ -1,4 +1,5 @@
 const redis = require("../redis");
+const utils = require("../utils");
 const knex = require("../knex");
 
 async function find(match) {
@@ -39,7 +40,7 @@ async function add(params) {
   if (id) {
     await knex("hosts").where("id", id).update({
       ...newHost,
-      updated_at: params.updated_at || new Date().toISOString()
+      updated_at: params.updated_at || utils.dateToUTC(new Date())
     });
   } else {
     // Mysql and sqlite don't support returning but return the inserted id by default

server/queries/ip.queries.js

@@ -1,5 +1,6 @@
 const { subMinutes } = require("date-fns");
 
+const utils = require("../utils");
 const knex = require("../knex");
 const env = require("../env");
 
@@ -9,7 +10,7 @@ async function add(ipToAdd) {
   const currentIP = await knex("ips").where("ip", ip).first();
   
   if (currentIP) {
-    const currentDate = new Date().toISOString();
+    const currentDate = utils.dateToUTC(new Date());
     await knex("ips")
       .where({ ip })
       .update({
@@ -41,7 +42,7 @@ function clear() {
   .where(
     "created_at",
     "<",
-    subMinutes(new Date(), env.NON_USER_COOLDOWN).toISOString()
+    utils.dateToUTC(subMinutes(new Date(), env.NON_USER_COOLDOWN))
   )
   .delete();
 }

server/queries/link.queries.js

@@ -1,9 +1,11 @@
 const bcrypt = require("bcryptjs");
 
-const CustomError = require("../utils").CustomError;
+const utils = require("../utils");
 const redis = require("../redis");
 const knex = require("../knex");
 
+const CustomError = utils.CustomError;
+
 const selectable = [
   "links.id",
   "links.address",
@@ -155,9 +157,9 @@ async function batchRemove(match) {
   
   const links = await findQuery;
   
-  links.forEach(redis.remove.link);
-  
   await deleteQuery.delete();
+  
+  links.forEach(redis.remove.link);
 }
 
 async function update(match, update) {
@@ -168,7 +170,7 @@ async function update(match, update) {
   
   await knex("links")
     .where(match)
-    .update({ ...update, updated_at: new Date().toISOString() });
+    .update({ ...update, updated_at: utils.dateToUTC(new Date()) });
 
   const links = await knex("links").select('*').where(match);
 

server/queries/user.queries.js

@@ -1,6 +1,7 @@
 const { addMinutes } = require("date-fns");
 const { v4: uuid } = require("uuid");
 
+const utils = require("../utils");
 const redis = require("../redis");
 const knex = require("../knex");
 
@@ -36,13 +37,13 @@ async function add(params, user) {
     email: params.email,
     password: params.password,
     verification_token: uuid(),
-    verification_expires: addMinutes(new Date(), 60).toISOString()
+    verification_expires: utils.dateToUTC(addMinutes(new Date(), 60))
   };
   
   if (user) {
     await knex("users")
       .where("id", user.id)
-      .update({ ...data, updated_at: new Date().toISOString() });
+      .update({ ...data, updated_at: utils.dateToUTC(new Date()) });
   } else {
     await knex("users").insert(data);
   }
@@ -69,7 +70,7 @@ async function update(match, update, methods) {
     });
   }
   
-  await updateQuery.update({ ...update, updated_at: new Date().toISOString() });
+  await updateQuery.update({ ...update, updated_at: utils.dateToUTC(new Date()) });
 
   const users = await query.select("*");
 

server/queries/visit.queries.js

@@ -1,4 +1,4 @@
-const { isAfter, subDays, subHours, set } = require("date-fns");
+const { isAfter, subDays, subHours, set, format } = require("date-fns");
 
 const utils = require("../utils");
 const redis = require("../redis");
@@ -11,10 +11,10 @@ async function add(params) {
     referrer: params.referrer.toLowerCase()
   };
 
-  const truncatedNow = new Date();
-  truncatedNow.setMinutes(0, 0, 0);
+  const nowUTC = new Date().toISOString();
+  const truncatedNow = nowUTC.substring(0, 10) + " " + nowUTC.substring(11, 14) + "00:00";
 
-  return  knex.transaction(async (trx) => {
+  return knex.transaction(async (trx) => {
     // Create a subquery first that truncates the
     const subquery = trx("visits")
       .select("visits.*")
@@ -27,26 +27,28 @@ async function add(params) {
     const visit = await trx
       .select("*")
       .from(subquery)
-      .where("created_at_hours", "=", truncatedNow.toISOString())
+      .where("created_at_hours", "=", truncatedNow)
       .forUpdate()
       .first();
-
+      
     if (visit) {
+      const countries = typeof visit.countries === "string" ? JSON.parse(visit.countries) : visit.countries;
+      const referrers = typeof visit.referrers === "string" ? JSON.parse(visit.referrers) : visit.referrers;
       await trx("visits")
         .where({ id: visit.id })
         .increment(`br_${data.browser}`, 1)
         .increment(`os_${data.os}`, 1)
         .increment("total", 1)
         .update({
-          updated_at: new Date().toISOString(),
-          countries: {
-            ...visit.countries,
-            [data.country]: (visit.countries[data.country] ?? 0) + 1
-          },
-          referrers: {
-            ...visit.referrers,
-             [data.referrer]: (visit.referrers[data.referrer] ?? 0) + 1
-          }
+          updated_at: utils.dateToUTC(new Date()),
+          countries: JSON.stringify({
+            ...countries,
+            [data.country]: (countries[data.country] ?? 0) + 1
+          }),
+          referrers: JSON.stringify({
+            ...referrers,
+             [data.referrer]: (referrers[data.referrer] ?? 0) + 1
+          })
         });
     } else {
       // This must also happen in the transaction to avoid concurrency
@@ -95,20 +97,21 @@ async function find(match, total) {
   };
 
   const visitsStream = knex("visits").where(match).stream();
-  const nowUTC = utils.getUTCDate();
   const now = new Date();
 
   const periods = utils.getStatsPeriods(now);
 
   for await (const visit of visitsStream) {
     periods.forEach(([type, fromDate]) => {
-      const isIncluded = isAfter(new Date(visit.created_at), fromDate);
+      const isIncluded = isAfter(utils.parseDatetime(visit.created_at), fromDate);
       if (!isIncluded) return;
       const diffFunction = utils.getDifferenceFunction(type);
-      const diff = diffFunction(now, new Date(visit.created_at));
+      const diff = diffFunction(now, utils.parseDatetime(visit.created_at));
       const index = stats[type].views.length - diff - 1;
       const view = stats[type].views[index];
       const period = stats[type].stats;
+      const countries = typeof visit.countries === "string" ? JSON.parse(visit.countries) : visit.countries;
+      const referrers = typeof visit.referrers === "string" ? JSON.parse(visit.referrers) : visit.referrers;
       stats[type].stats = {
         browser: {
           chrome: period.browser.chrome + visit.br_chrome,
@@ -129,7 +132,7 @@ async function find(match, total) {
         },
         country: {
           ...period.country,
-          ...Object.entries(visit.countries).reduce(
+          ...Object.entries(countries).reduce(
             (obj, [country, count]) => ({
               ...obj,
               [country]: (period.country[country] || 0) + count
@@ -139,7 +142,7 @@ async function find(match, total) {
         },
         referrer: {
           ...period.referrer,
-          ...Object.entries(visit.referrers).reduce(
+          ...Object.entries(referrers).reduce(
             (obj, [referrer, count]) => ({
               ...obj,
               [referrer]: (period.referrer[referrer] || 0) + count
@@ -174,7 +177,7 @@ async function find(match, total) {
       views: stats.lastWeek.views,
       total: stats.lastWeek.total
     },
-    updatedAt: new Date().toISOString()
+    updatedAt: new Date()
   };
 
   if (match.link_id) {

server/utils/utils.js

@@ -2,6 +2,7 @@ const { differenceInDays, differenceInHours, differenceInMonths, differenceInMil
 const nanoid = require("nanoid/generate");
 const knexUtils = require("./knex");
 const JWT = require("jsonwebtoken");
+const knex = require("../knex");
 const path = require("path");
 const hbs = require("hbs");
 const ms = require("ms");
@@ -113,14 +114,27 @@ function getDifferenceFunction(type) {
   throw new Error("Unknown type.");
 }
 
-function getUTCDate(dateString) {
-  const date = new Date(dateString || Date.now());
-  return new Date(
-    date.getUTCFullYear(),
-    date.getUTCMonth(),
-    date.getUTCDate(),
-    date.getUTCHours()
-  );
+function parseDatetime(date) {
+  // because postgres returns date, sqlite returns iso 8601 string in utc
+  return date instanceof Date ? date : new Date(date + "Z");
+}
+
+function parseTimestamps(item) {
+  return {
+    created_at: parseDatetime(item.created_at),
+    updated_at: parseDatetime(item.updated_at),
+  }
+}
+
+function dateToUTC(date) {
+  const dateUTC = date instanceof Date ? date.toISOString() : new Date(date).toISOString();
+  
+  // sqlite needs iso 8601 string in utc
+  if (knex.isSqlite3) {
+    return dateUTC.substring(0, 10) + " " + dateUTC.substring(11, 19);
+  };
+  
+  return dateUTC;
 }
 
 function getStatsPeriods(now) {
@@ -197,7 +211,8 @@ const MINUTE = 60,
       WEEK = DAY * 7,
       MONTH = DAY * 30,
       YEAR = DAY * 365;
-function getTimeAgo(date) {
+function getTimeAgo(dateString) {
+  const date = new Date(dateString);
   const secondsAgo = Math.round((Date.now() - Number(date)) / 1000);
 
   if (secondsAgo < MINUTE) {
@@ -229,23 +244,28 @@ function getTimeAgo(date) {
 const sanitize = {
   domain: domain => ({
     ...domain,
+    ...parseTimestamps(domain),
     id: domain.uuid,
     uuid: undefined,
     user_id: undefined,
     banned_by_id: undefined
   }),
-  link: link => ({
-    ...link,
-    banned_by_id: undefined,
-    domain_id: undefined,
-    user_id: undefined,
-    uuid: undefined,
-    id: link.uuid,
-    relative_created_at: getTimeAgo(link.created_at),
-    relative_expire_in: link.expire_in && ms(differenceInMilliseconds(new Date(link.expire_in), new Date()), { long: true }),
-    password: !!link.password,
-    link: getShortURL(link.address, link.domain)
-  })
+  link: link => {
+    const timestamps = parseTimestamps(link);
+    return {
+      ...link,
+      ...timestamps,
+      banned_by_id: undefined,
+      domain_id: undefined,
+      user_id: undefined,
+      uuid: undefined,
+      id: link.uuid,
+      relative_created_at: getTimeAgo(timestamps.created_at),
+      relative_expire_in: link.expire_in && ms(differenceInMilliseconds(parseDatetime(link.expire_in), new Date()), { long: true }),
+      password: !!link.password,
+      link: getShortURL(link.address, link.domain)
+    }
+  }
 };
 
 function sleep(ms) {
@@ -286,6 +306,7 @@ function registerHandlebarsHelpers() {
 module.exports = {
   addProtocol,
   CustomError,
+  dateToUTC,
   deleteCurrentToken,
   generateId,
   getDifferenceFunction,
@@ -295,8 +316,9 @@ module.exports = {
   getStatsCacheTime,
   getStatsLimit,
   getStatsPeriods,
-  getUTCDate,
   isAdmin,
+  parseDatetime,
+  parseTimestamps,
   preservedURLs,
   registerHandlebarsHelpers,
   removeWww,