A Better Structure For Express/node.js Apps
If you’d like to start a new Express app, there are thousands of articles on how to do so. Most of them, targeting beginners, include all the code in one or two files. Although, technically correct, these articles are quite lax on good practices. In no particular order, I’d like to address the following issues.
App Start Up Issues
The API/web endpoint accepts requests before a database connection was established. Usually, the offending code would be structured like this:
// app.js
var http = require('http');
var app = express();
// ...
mongoose.connect('mongodb://localhost/db');
// ...
app.use(require('./routes'));
// ...
var server = app.listen( process.env.PORT || 3000, function(){
console.log(`Listening on port ${server.address().port}`);
});
Because, in node.js, almost everything is asynchronous, your app will start listening for connections before the database connection was established. Although this might not be a problem when developing the app, it’s a different story when deployed in production. A fast start up time might be 300-500 ms, connecting to redis and mongodb, might take 600 ms or more. Are you sure there are no incoming connections in those 300-600 ms your app takes to start?
A minimal fix, is to start listening for connections once the database connection was established.
mongoose.connection.on('open', function(){
var server = app.listen( process.env.PORT || 3000, function(){
console.log(`Listening on port ${server.address().port}`);
});
});
No Setup For Connecting to Multiple Databases/Services
Most examples connect to zero
app.get('/', function(req, res, next){
return res.send('Hello World!');
});
or one database/service.
// routes/articles.js
var router = require('express').Router();
var mongoose = require('mongoose');
var Article = mongoose.model('Article');
router.get('/', auth.optional, function(req, res, next) {
Article.find({}, function(err, articles){
if (err) {
return next(err);
}
return res.json(articles || []);
});
});
To address this issue, I would change a couple of things. First, your router should not require your model. If really what to do that query in your router, at minimum, you should pass in the router and model as a dependency.
// routes/articles.js
module.exports = function(dependencies) {
var router = dependencies.router;
var Article = dependencies.Article;
// var redisClient = dependencies.redisClient;
router.get('/', function(req, res, next) {
Article.find({}, function(err, articles){
if (err) {
return next(err);
}
return res.json(articles || []);
});
});
};
// app.js
var mongoose = require('mongoose');
var dependencies = {
router: require('express').Router(),
Article: mongoose.model('Article')
};
var articlesRoute = require('routes/articles')(dependencies);
app.use(articlesRoute);
At least, you’ve got a chance to mock router and/or the model, to test them independently.
If one would like to go further, model and/or the redis client can be further refactored into a store.js file that will handle the databases interaction. For example, return a cached version of the articles, and only if none is found, query the database. Also, store the queried result for 10 seconds.
// lib/store.js
module.exports = function(dependencies) {
var Article = dependencies.Article;
var redisClient = dependencies.redisClient;
function articles(callback) {
return redisClient.get('articles', function(err, items){
// if err ... unhandled
if (items) {
return callback(err, items);
}
Article.find({}, function(err, articles){
// if err ...
var ttl = 10; // expire cache after 10 seconds
return redisCLient.set('articles', articles, 'ex', ttl,
function(err, result){
return callback(err, articles);
});
});
});
}
return {
articles: articles
}
}
// app.js
var Article; // from require or dependencies
var redisClient;
var store = require('lib/store')({
Article,
redisClient
});
var articlesRoute = require('routes/articles')({
router: require('express').Router(),
store
});
// routes/articles.js
module.exports = function(dependencies) {
var store = dependencies.store;
router.get('/', function(req, res, next) {
return store.articles(function(err, items){
if (err) {
return next(err);
}
return res.json(articles || []);
})
}
}
To write a unit test for the above code, one would start with store.js. The npm packages mockgoose and redis-js could be used to setup an in-memory database for mongo and redis.
Using the same technique, one can inject any type of service that requires configuration and setup (eg. RabbitMQ, Push Notifications Services, websockets). If at a later point, one would like to replace redis with memcache, there shouldn’t be a require('redis') in every file.
Tight Code Coupling
Or as seen in almost every online example require the same local or external module in every file.
// router.js
var mongoose = require('mongoose');
var Article = mongoose.model('Article');
can be replaced with
// router.js
module.exports = function(dependencies) {
var Article = dependencies.Article;
// var models = dependencies.models; // or better yet
}
The same can be applied to app.js.
// app.js
module.exports = function(dependencies) {
var config = dependencies.config;
var models = dependencies.models;
var redisClient = dependencies.redisClient;
var app = express();
// ...
return app;
}
And even to config.js. Beside all the connection strings (mongo, redis, etc), one can use this file to assign a release version to the current app. This maybe be useful if one needs to pass in that info to other services, like sentry.io.
// config.js
module.exports = function(dependencies) {
var process = dependencies.process;
var os = dependencies.os;
var p = require('../package.json'); // or from dependencies
var config = {
port: process.env.NODE_PORT || 3000,
hostname: process.env.NODE_PORT || os.hostname(),
mongoUrl: process.env.NODE_MONGO_URL || 'mongodb://localhost/db',
release: p.name + '-' + p.version
};
return config;
}
Using the presented building blocks, when setting up an app, a start module can be build, so all the services start only after the required connections were established.
// lib/start.js
module.exports = function(dependencies) {
var config = dependencies.config;
var process = dependencies.process;
var mongoose = dependencies.mongoose;
function redisOnConnect(startTime, mongooseOnOpen) {
return function redisOnConnectFunc() {
console.log(`Redis connected after ${(new Date() - startTime)} ms.`);
mongoose.connect(config.mongoUrl, {
useMongoClient: true
});
mongoose.connection.on('error', onError);
mongoose.connection.on('disconnected', function(){
debug('Mongoose connection disconnected');
process.exit();
});
mongoose.connection.on('open', mongooseOnOpen(config, startTime));
};
};
function onError(error) {
console.log('Connection error: ' + err);
process.exit();
}
function serverOnListening(startTime, server) {
return function onListeningFunc() {
var address = server.address();
console.log(`Server listening on ${address.address}:${address.port}.`);
console.log(`Start time: ${(new Date() - startTime)} ms.`);
};
}
return {
redisOnConnect,
onError,
serverOnListening
};
}
// bin/service.js
var mongoose = require('mongoose');
var config = require('lib/config')({
process,
os: require('os')
});
var start = require('lib/start')({
config,
process,
mongoose
});
function mongooseOnOpen(config, startTime) {
return function mongooseOnOpenFunc() {
console.log(`Mongoose connection open in ${(new Date() - startTime)} ms.`);
var app = require('../app')({
config,
store
});
var http = require('http');
var server = http.createServer(app);
server.listen(config.port);
server.on('error', start.onError);
server.on('listening', start.serverOnListening(startTime, server));
};
};
var redisClient = redis.client(config);
redisClient.on('ready', start.redisOnConnect(startTime, mongooseOnOpen));
As an exercise, when writing a new service, one should answer the question, how should this new code be tested? Today, all the cool kids use mongo, but tomorrow, they go hipster and switch back to FoxPro.