Search This Blog

2024/04/29

MongoDb:GridFs uploading file to mongodb using multer

We know that it is possible to upload a file into DMBS like Mysql or MSSQL.
Today we will explore how to upload file to mongodb.

First create a express application as follows

express --view=ejs express-mongoose-gridfs

run
npm i

to install packageds.

Now we need

crypto,mongodb,mongoose,multer-gridfs-storage install them one by one.

Here is my package.json

{
"name": "express-mongoose-gridfs",
"version": "0.0.0",
"private": true,
"scripts": {
"start": "node ./bin/www"
},
"dependencies": {
"cookie-parser": "~1.4.4",
"crypto": "^1.0.1",
"debug": "~2.6.9",
"ejs": "~2.6.1",
"express": "~4.16.1",
"http-errors": "~1.6.3",
"mongodb": "^5.9.1",
"mongoose": "^8.3.2",
"morgan": "~1.9.1",
"multer-gridfs-storage": "^5.0.2"
}
}

Note that "mongodb": "^5.9.1", is essential for higher version code failed.

create connection.js at root of project with following content

let mongoose = require("mongoose");
let Schema = mongoose.Schema;
const GridFsStorage = require("multer-gridfs-storage");

const mongoURI =
"mongodb://sangram:sangram%2381@127.0.0.1:27017
/phenixDb?directConnection=true&serverSelectionTimeoutMS=2000
&authSource=admin&appName=mongosh+2.2.3";

const conn = mongoose.createConnection(mongoURI, { maxPoolSize: 10 });

//initialize gridfs
let bucket = new mongoose.mongo.GridFSBucket(conn, {
bucketName: "uploads",
});

module.exports = {
mongoose: mongoose,
connection: conn,
Schema: Schema,
mongoURI: mongoURI,
bucket: bucket,
};


create upload.js in root of project with following content.

const multer = require("multer");
const {GridFsStorage} = require("multer-gridfs-storage");
const crypto = require("crypto");
const {mongoURI} = require("./connection")
const path = require("path");

const storage = new GridFsStorage({
url: mongoURI,
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
const filename = buf.toString("hex") + path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: "uploads",
};
resolve(fileInfo);
});
});
},
});

const fileFilter = (req, file, cb) => {
if (file.fieldname == "profilePic") {
if ((file.mimetype).includes('jpeg')
|| (file.mimetype).includes('png')
|| (file.mimetype).includes('jpg')) {
cb(null, true);
} else {
cb(null, false);
}
}
if (file.fieldname == "resume") {
if ((file.mimetype).includes('doc')
|| (file.mimetype).includes('openxmlformats')) {
cb(null, true);
} else {
cb(null, false);
}
}
};

var upload = multer({ storage: storage, fileFilter: fileFilter,
limits: { fileSize: 1 * 1024 * 1024 } })

module.exports = {
upload: upload,
storage:storage
}

create foodItem.js in models folder at root of project with following
content

const {mongoose,Schema,connection} = require("../connection")

let foodItemSchema = new Schema({
name: String,
price: Number,
description: String,
picture : String,
createdOn: { type: Date, default: Date.now },
isDeleted: { type: Boolean, default: false },
});

//last one is actual collection name in db and its essential
let FoodItem = connection.model("FoodItem", foodItemSchema, "foodItem");

module.exports = {
FoodItem: FoodItem,
FoodItemSchema: foodItemSchema,
};


create demo.js in route folder with following content.

let express = require("express");
let router = express.Router();
let FoodItem = require("../models/foodItem").FoodItem;
let FoodItemSchema = require("../models/foodItem").FoodItemSchema;
let mongoose = require("mongoose");
let {upload} = require("../upload");
const path = require('path')
const fs = require('fs')
let {bucket} = require("../connection")

/* insert a new food item */
router.post("/insert",upload.single('resume'), async function (req, res, next) {
let foodItemName = req.body.name;
let foodItemDesc = req.body.description;
let foodItemPrice = req.body.price;

console.log("Files",)

try {
let foodItem = await FoodItem.create({
name: foodItemName,
description: foodItemDesc,
price: foodItemPrice,
picture:req.file.filename
});
res.json({
message: "food item added successfully",
error: "",
data: foodItem,
success: true,
});
} catch (err) {
res.json({
message: "unable to save food items",
error: err.message.toString(),
data: [],
success: false,
});
}
});


router.get("/image/:filename", async (req, res) => {
let foods = await FoodItem.find({ picture: req.params.filename })
if(foods.length){
bucket.openDownloadStreamByName(req.params.filename).
pipe(res);
}else{
res.json({"msg":"no file found"})
}
});



module.exports = router;


inside app.js

add
var demoRouter = require('./routes/demo');

and
app.use('/demo', demoRouter);

at suitable place

You can start your application using

npm start

Now you can test your application using postman

Testing Insert API:

curl --location 'http://localhost:3000/demo/insert' \
--form 'name=" Anda Burji\""' \
--form 'description=" made up of smashed eggs"' \
--form 'price=" 40"' \
--form 'resume=@"/home/sangram/Downloads/Cv-Sangram.docx"'

Output:
{
"message": "food item added successfully",
"error": "",
"data": {
"name": " Anda Burji\"",
"price": 40,
"description": " made up of smashed eggs",
"picture": "2dc4d730d659e9cd14d6c0506ada7ef9.docx",
"isDeleted": false,
"_id": "662f6c080fd2bdd1e18990bf",
"createdOn": "2024-04-29T09:44:40.222Z",
"__v": 0
},
"success": true
}

Please note picture value though its not image but file.

Testing Download API:
here we are using value of picture noted above,this should download the file

curl --location
'http://localhost:3000/demo/image/2dc4d730d659e9cd14d6c0506ada7ef9.docx'


Code of this application is available at
https://github.com/gitsangramdesai/express-mongoose-gridfs.

2024/04/28

Javascript:Validate JSON against schema

As we know when we have given a xml we can can validate it ,if it follows the
format we are looking far using xsd,similarly there is need for validatio of
json, we check given json is valid json by using JSON.parse() method but it
does not let us to check if json confirms to our own format so here is
library called ajv that precisely do that.

Code 1:
const Ajv = require("ajv")
const ajv = new Ajv()

const schema = {
type: "object",
properties: {
foo: {type: "integer"},
bar: {type: "string"}
},
required: ["foo"],
additionalProperties: false
}
const validate = ajv.compile(schema)

const data = {
foo: 1,
bar: "abc",
add:23
}
const valid = validate(data)
if (!valid) {
console.log(validate.errors)
}else{
console.log("Valid JSON")
}

Output:
[
{
instancePath: '',
schemaPath: '#/additionalProperties',
keyword: 'additionalProperties',
params: { additionalProperty: 'add' },
message: 'must NOT have additional properties'
}
]

Here our json string has additional property called add
so our json is not valid wrt our schema.


Same library can parse the json and it can do same work as
JSON.parse & also it can serialize json object into string.

Code 2:

const Ajv = require("ajv/dist/jtd")
const ajv = new Ajv() // options can be passed, e.g. {allErrors: true}

const schema = {
properties: {
foo: {type: "int32"}
},
optionalProperties: {
bar: {type: "string"}
}
}

const serialize = ajv.compileSerializer(schema)

const data = {
foo: 1,
bar: "abc"
}

console.log("Serializer:",serialize(data))

const parse = ajv.compileParser(schema)

const json = '{"foo": 1, "bar": "abc"}'
const invalidJson = '{"unknown": "abc"}'

parseAndLog(json) // logs {foo: 1, bar: "abc"}
parseAndLog(invalidJson) // logs error and position

function parseAndLog(json) {
const data = parse(json)
if (data === undefined) {
console.log("Parsing Failed:")
console.log(parse.message) // error message from the last parse call
console.log(parse.position) // error position in string
} else {
console.log("Parsing Success:")
console.log(data)
}
}

Output:
Serializer: {"foo":1,"bar":"abc"}
Parsing Success:
{ foo: 1, bar: 'abc' }
Parsing Failed:
property unknown not allowed
11

2024/04/25

MongoDb:$facet


 $facet:

Processes multiple aggregation pipelines within a single stage on the
same set of input documents. Each sub-pipeline has its own field in the
output document where its results are stored as an array of documents.

The $facet stage allows you to create multi-faceted aggregations which
characterize data across multiple dimensions, or facets, within a single
aggregation stage. Multi-faceted aggregations provide multiple filters and
categorizations to guide data browsing and analysis. Retailers commonly use
faceting to narrow search results by creating filters on product price,
manufacturer, size, etc.

Input documents are passed to the $facet stage only once. $facet enables
various aggregations on the same set of input documents, without needing to
retrieve the input documents multiple times.

We wil use teacher collection for understanding $facet.

//drop teacher collection if prexist
db.teacher.drop()

//add some sample data for teacher collection
db.teacher.insertMany([
{
name:"sagar desai",
email:"sagar.desai@example.com",
age:30,
gender:"male",
},
{
name:"sachin sawant",
email:"sagar.sawant@example.com",
age:30,
gender:"male",
},
{
name:"swara desai",
email:"swara.desai@example.com",
age:20,
gender:"female",
},
{
name:"mrunmayi sawant",
email:"mrunmayi.sawant@example.com",
age:30,
gender:"female",
},
{
name:"sandesh raorane",
email:"sandesh.raorane@example.com",
age:45,
gender:"male",
},
{
name:"Gaurav raorane",
email:"gaurav.raorane@example.com",
age:42,
gender:"male",
},
{
name:"vinayak tambe",
email:"vinayak.tambe@example.com",
age:21,
gender:"male",
},
{
name:"kiran rao",
email:"kiran.rao@example.com",
age:21,
gender:"trans-gender",
}
])


Lets first run following command
db.teacher.aggregate([
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1}
}
}
}
])

Output:
[
{ _id: 0, count: 3 },
{ _id: 30, count: 3 },
{ _id: 'GreaterThan40', count: 2 }
]

db.teacher.aggregate([
{
$group:{
_id:"$gender",
count:{"$sum":1}
}
}
])
Output:
[
{ _id: 'female', count: 2 },
{ _id: 'trans-gender', count: 1 },
{ _id: 'male', count: 5 }
]


Ouput of last two queries can be done in single query using facet.
Here is how


db.teacher.aggregate([
{
$facet:{
"facetByAge":[
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1}
}
}
}
],
"faceByGender":[{
$group:{
_id:"$gender",
count:{"$sum":1}
}
}]
}
}
])

Output:
[
{
facetByAge: [
{ _id: 0, count: 3 },
{ _id: 30, count: 3 },
{ _id: 'GreaterThan40', count: 2 }
],
faceByGender: [
{ _id: 'male', count: 5 },
{ _id: 'female', count: 2 },
{ _id: 'trans-gender', count: 1 }
]
}
]

We can get all document based on grouping by gender & age just
my small modification in our previous query.

db.teacher.aggregate([
{
$facet:{
"facetByAge":[
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1},
teachers:{$push:{"email":"$email",
            "age":"$age",name:"$name",gender:"$gender"}}
}
}
},{
$project:{
_id:1,
count:1,
teachers:1,
}
}
],
"faceByGender":[{
$group:{
_id:"$gender",
count:{"$sum":1},
teachers:{$push:{"email":"$email",
            "age":"$age",name:"$name",gender:"$gender"}}
}
}]
}
}
])

Output:
[
{
facetByAge: [
{
_id: 0,
count: 3,
teachers: [
{
email: 'swara.desai@example.com',
age: 20,
name: 'swara desai',
gender: 'female'
},
{
email: 'vinayak.tambe@example.com',
age: 21,
name: 'vinayak tambe',
gender: 'male'
},
{
email: 'kiran.rao@example.com',
age: 21,
name: 'kiran rao',
gender: 'trans-gender'
}
]
},
{
_id: 30,
count: 3,
teachers: [
{
email: 'sagar.desai@example.com',
age: 30,
name: 'sagar desai',
gender: 'male'
},
{
email: 'sagar.sawant@example.com',
age: 30,
name: 'sachin sawant',
gender: 'male'
},
{
email: 'mrunmayi.sawant@example.com',
age: 30,
name: 'mrunmayi sawant',
gender: 'female'
}
]
},
{
_id: 'GreaterThan40',
count: 2,
teachers: [
{
email: 'sandesh.raorane@example.com',
age: 45,
name: 'sandesh raorane',
gender: 'male'
},
{
email: 'gaurav.raorane@example.com',
age: 42,
name: 'Gaurav raorane',
gender: 'male'
}
]
}
],
faceByGender: [
{
_id: 'male',
count: 5,
teachers: [
{
email: 'sagar.desai@example.com',
age: 30,
name: 'sagar desai',
gender: 'male'
},
{
email: 'sagar.sawant@example.com',
age: 30,
name: 'sachin sawant',
gender: 'male'
},
{
email: 'sandesh.raorane@example.com',
age: 45,
name: 'sandesh raorane',
gender: 'male'
},
{
email: 'gaurav.raorane@example.com',
age: 42,
name: 'Gaurav raorane',
gender: 'male'
},
{
email: 'vinayak.tambe@example.com',
age: 21,
name: 'vinayak tambe',
gender: 'male'
}
]
},
{
_id: 'female',
count: 2,
teachers: [
{
email: 'swara.desai@example.com',
age: 20,
name: 'swara desai',
gender: 'female'
},
{
email: 'mrunmayi.sawant@example.com',
age: 30,
name: 'mrunmayi sawant',
gender: 'female'
}
]
},
{
_id: 'trans-gender',
count: 1,
teachers: [
{
email: 'kiran.rao@example.com',
age: 21,
name: 'kiran rao',
gender: 'trans-gender'
}
]
}
]
}
]

In this output

_id =0 corresponds with age < 30
_id =30 corresponds with 30 <= age < 40
_id =0 corresponds with age > 40

accordingly we will use switch case to
give that in result.Below is modified query

db.teacher.aggregate([
{
$facet:{
"facetByAge":[
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1},
teachers:{$push:{"email":"$email",
            "age":"$age",name:"$name",gender:"$gender"}}
}
}
},{
$project:{
_id:1,
count:1,
teachers:1,
"AgeGroup" :{
$switch:
{
branches: [
{
case: { $eq :["$_id", 0] },
then: "age < 30"
},
{
case: { $eq:["$_id",30] },
then: "30 <= age < 40"
}
],
default: "age > 40"
}
}
}
}
],
"faceByGender":[{
$group:{
_id:"$gender",
count:{"$sum":1},
teachers:{$push:{"email":"$email",
            "age":"$age",name:"$name",gender:"$gender"}}
}
}]
}
}
])

Output:
[
{
facetByAge: [
{
_id: 0,
count: 3,
teachers: [
{
email: 'swara.desai@example.com',
age: 20,
name: 'swara desai',
gender: 'female'
},
{
email: 'vinayak.tambe@example.com',
age: 21,
name: 'vinayak tambe',
gender: 'male'
},
{
email: 'kiran.rao@example.com',
age: 21,
name: 'kiran rao',
gender: 'trans-gender'
}
],
AgeGroup: 'age < 30'
},
{
_id: 30,
count: 3,
teachers: [
{
email: 'sagar.desai@example.com',
age: 30,
name: 'sagar desai',
gender: 'male'
},
{
email: 'sagar.sawant@example.com',
age: 30,
name: 'sachin sawant',
gender: 'male'
},
{
email: 'mrunmayi.sawant@example.com',
age: 30,
name: 'mrunmayi sawant',
gender: 'female'
}
],
AgeGroup: '30 <= age < 40'
},
{
_id: 'GreaterThan40',
count: 2,
teachers: [
{
email: 'sandesh.raorane@example.com',
age: 45,
name: 'sandesh raorane',
gender: 'male'
},
{
email: 'gaurav.raorane@example.com',
age: 42,
name: 'Gaurav raorane',
gender: 'male'
}
],
AgeGroup: 'age > 40'
}
],
faceByGender: [
{
_id: 'male',
count: 5,
teachers: [
{
email: 'sagar.desai@example.com',
age: 30,
name: 'sagar desai',
gender: 'male'
},
{
email: 'sagar.sawant@example.com',
age: 30,
name: 'sachin sawant',
gender: 'male'
},
{
email: 'sandesh.raorane@example.com',
age: 45,
name: 'sandesh raorane',
gender: 'male'
},
{
email: 'gaurav.raorane@example.com',
age: 42,
name: 'Gaurav raorane',
gender: 'male'
},
{
email: 'vinayak.tambe@example.com',
age: 21,
name: 'vinayak tambe',
gender: 'male'
}
]
},
{
_id: 'female',
count: 2,
teachers: [
{
email: 'swara.desai@example.com',
age: 20,
name: 'swara desai',
gender: 'female'
},
{
email: 'mrunmayi.sawant@example.com',
age: 30,
name: 'mrunmayi sawant',
gender: 'female'
}
]
},
{
_id: 'trans-gender',
count: 1,
teachers: [
{
email: 'kiran.rao@example.com',
age: 21,
name: 'kiran rao',
gender: 'trans-gender'
}
]
}
]
}
]

Usually in aggregations output of previous operation is input to
next operation,but in facet all section get same input.

MongoDb: $sortByCount operator

$sortByCount:
Groups incoming documents based on the value of a specified expression,
then computes the count of documents in each distinct group.

Each output document contains two fields: an _id field containing the
distinct grouping value, and a count field containing the number of
documents belonging to that grouping or category.

The documents are sorted by count in descending order.

db.exhibits.insertMany([
{ "_id" : 1, "title" : "The Pillars of Society", "artist" : "Grosz", "year" : 1926, "tags" : [ "painting", "satire", "Expressionism", "caricature" ] },
{ "_id" : 2, "title" : "Melancholy III", "artist" : "Munch", "year" : 1902, "tags" : [ "woodcut", "Expressionism" ] },
{ "_id" : 3, "title" : "Dancer", "artist" : "Miro", "year" : 1925, "tags" : [ "oil", "Surrealism", "painting" ] },
{ "_id" : 4, "title" : "The Great Wave off Kanagawa", "artist" : "Hokusai", "tags" : [ "woodblock", "ukiyo-e" ] },
{ "_id" : 5, "title" : "The Persistence of Memory", "artist" : "Dali", "year" : 1931, "tags" : [ "Surrealism", "painting", "oil" ] },
{ "_id" : 6, "title" : "Composition VII", "artist" : "Kandinsky", "year" : 1913, "tags" : [ "oil", "painting", "abstract" ] },
{ "_id" : 7, "title" : "The Scream", "artist" : "Munch", "year" : 1893, "tags" : [ "Expressionism", "painting", "oil" ] },
{ "_id" : 8, "title" : "Blue Flower", "artist" : "O'Keefe", "year" : 1918, "tags" : [ "abstract", "painting" ] }
])


db.exhibits.aggregate( [
{ $unwind: "$tags" }
])

Output:
[
{
_id: 1,
title: 'The Pillars of Society',
artist: 'Grosz',
year: 1926,
tags: 'painting'
},
{
_id: 1,
title: 'The Pillars of Society',
artist: 'Grosz',
year: 1926,
tags: 'satire'
},
{
_id: 1,
title: 'The Pillars of Society',
artist: 'Grosz',
year: 1926,
tags: 'Expressionism'
},
{
_id: 1,
title: 'The Pillars of Society',
artist: 'Grosz',
year: 1926,
tags: 'caricature'
},
{
_id: 2,
title: 'Melancholy III',
artist: 'Munch',
year: 1902,
tags: 'woodcut'
},
{
_id: 2,
title: 'Melancholy III',
artist: 'Munch',
year: 1902,
tags: 'Expressionism'
},
{ _id: 3, title: 'Dancer', artist: 'Miro', year: 1925, tags: 'oil' },
{
_id: 3,
title: 'Dancer',
artist: 'Miro',
year: 1925,
tags: 'Surrealism'
},
{
_id: 3,
title: 'Dancer',
artist: 'Miro',
year: 1925,
tags: 'painting'
},
{
_id: 4,
title: 'The Great Wave off Kanagawa',
artist: 'Hokusai',
tags: 'woodblock'
},
{
_id: 4,
title: 'The Great Wave off Kanagawa',
artist: 'Hokusai',
tags: 'ukiyo-e'
},
{
_id: 5,
title: 'The Persistence of Memory',
artist: 'Dali',
year: 1931,
tags: 'Surrealism'
},
{
_id: 5,
title: 'The Persistence of Memory',
artist: 'Dali',
year: 1931,
tags: 'painting'
},
{
_id: 5,
title: 'The Persistence of Memory',
artist: 'Dali',
year: 1931,
tags: 'oil'
},
{
_id: 6,
title: 'Composition VII',
artist: 'Kandinsky',
year: 1913,
tags: 'oil'
},
{
_id: 6,
title: 'Composition VII',
artist: 'Kandinsky',
year: 1913,
tags: 'painting'
},
{
_id: 6,
title: 'Composition VII',
artist: 'Kandinsky',
year: 1913,
tags: 'abstract'
},
{
_id: 7,
title: 'The Scream',
artist: 'Munch',
year: 1893,
tags: 'Expressionism'
},
{
_id: 7,
title: 'The Scream',
artist: 'Munch',
year: 1893,
tags: 'painting'
},
{
_id: 7,
title: 'The Scream',
artist: 'Munch',
year: 1893,
tags: 'oil'
}
]
Actual query:

Gives count based on tags & sort documents
descending order based on count.

db.exhibits.aggregate( [
{ $unwind: "$tags" },
{ $sortByCount: "$tags" }
])


Output:
[
{ _id: 'painting', count: 6 },
{ _id: 'oil', count: 4 },
{ _id: 'Expressionism', count: 3 },
{ _id: 'abstract', count: 2 },
{ _id: 'Surrealism', count: 2 },
{ _id: 'ukiyo-e', count: 1 },
{ _id: 'woodcut', count: 1 },
{ _id: 'woodblock', count: 1 },
{ _id: 'caricature', count: 1 },
{ _id: 'satire', count: 1 }
]

MongoDb:$bucket operator

Today we will explore $bucket operator in mongodb.

$bucket:
Categorizes incoming documents into groups, called buckets, based on a
specified expression and bucket boundaries and outputs a document per each
bucket. Each output document contains an _id field whose value specifies the
inclusive lower bound of the bucket. The output option specifies the fields
included in each output document.

$bucket only produces output documents for buckets that contain at least one
input document.

we will create a collection teacher for this purpose.

//drop teacher collection if exists
db.teacher.drop()

//populate teacher collection with data.
db.teacher.insertMany([
{
name:"sagar desai",
email:"sagar.desai@example.com",
age:30,
gender:"male",
},
{
name:"sachin sawant",
email:"sagar.sawant@example.com",
age:30,
gender:"male",
},
{
name:"swara desai",
email:"swara.desai@example.com",
age:20,
gender:"female",
},
{
name:"mrunmayi sawant",
email:"mrunmayi.sawant@example.com",
age:30,
gender:"female",
},
{
name:"sandesh raorane",
email:"sandesh.raorane@example.com",
age:45,
gender:"male",
},
{
name:"Gaurav raorane",
email:"gaurav.raorane@example.com",
age:42,
gender:"male",
},
{
name:"vinayak tambe",
email:"vinayak.tambe@example.com",
age:21,
gender:"male",
},
{
name:"kiran rao",
email:"kiran.rao@example.com",
age:21,
gender:"trans-gender",
}
])

This will give us counts based on bucket.
For Age :0-30 (0 included,30 excluded) ,total count of teachers
For Age :30-40 (30 included,40 excluded) ,total count of teachers
For Age :40-more(40 included,more than 40)),total count of teachers

db.teacher.aggregate([
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1}
}
}
}
])

Output:
[
{ _id: 0, count: 3 },
{ _id: 30, count: 3 },
{ _id: 'GreaterThan40', count: 2 }
]

Below query along with previous output adds emails of teacher into array email
array for each group formed by boundries

db.teacher.aggregate([
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1},
email:{$push:"$email"}
}
}
}
])

Output:
[
{
_id: 0,
count: 3,
email: [
'swara.desai@example.com',
'vinayak.tambe@example.com',
'kiran.rao@example.com'
]
},
{
_id: 30,
count: 3,
email: [
'sagar.desai@example.com',
'sagar.sawant@example.com',
'mrunmayi.sawant@example.com'
]
},
{
_id: 'GreaterThan40',
count: 2,
email: [ 'sandesh.raorane@example.com', 'gaurav.raorane@example.com' ]
}
]

Below query along with counts for each group adds total documents in teachers
array.

db.teacher.aggregate([
{
$bucket:{
groupBy:"$age",
boundaries:[0,30,40],
default:"GreaterThan40",
output:{
count:{"$sum":1},
teachers:{$push:{"email":"$email","age":"$age",
name:"$name",gender:"$gender"}}
}
}
}
])

Output:
[
{
_id: 0,
count: 3,
teachers: [
{
email: 'swara.desai@example.com',
age: 20,
name: 'swara desai',
gender: 'female'
},
{
email: 'vinayak.tambe@example.com',
age: 21,
name: 'vinayak tambe',
gender: 'male'
},
{
email: 'kiran.rao@example.com',
age: 21,
name: 'kiran rao',
gender: 'trans-gender'
}
]
},
{
_id: 30,
count: 3,
teachers: [
{
email: 'sagar.desai@example.com',
age: 30,
name: 'sagar desai',
gender: 'male'
},
{
email: 'sagar.sawant@example.com',
age: 30,
name: 'sachin sawant',
gender: 'male'
},
{
email: 'mrunmayi.sawant@example.com',
age: 30,
name: 'mrunmayi sawant',
gender: 'female'
}
]
},
{
_id: 'GreaterThan40',
count: 2,
teachers: [
{
email: 'sandesh.raorane@example.com',
age: 45,
name: 'sandesh raorane',
gender: 'male'
},
{
email: 'gaurav.raorane@example.com',
age: 42,
name: 'Gaurav raorane',
gender: 'male'
}
]
}
]

We can not use string with boundry condition as the comparison is lexographic.

Lets do it by another way

We build this query step by step.

Step 1:
db.teacher.aggregate([
{
$group:{
_id:"$gender",
count:{"$sum":1}
}
}
])



is logically almost equivalent to

db.teacher.aggregate([
{
$group:{
_id:{"gender":"$gender"},
count:{"$sum":1}
}
}
])

Output:
[
{ _id: 'female', count: 2 },
{ _id: 'trans-gender', count: 1 },
{ _id: 'male', count: 5 }
]


Now


db.teacher.aggregate([
{
$group:{
_id:{"gender":"$gender"},
count:{"$sum":1},
teachers:{$push:{"email":"$email","age":"$age",
                name:"$name",gender:"$gender"}}
}
}
])

Output:
[
{
_id: { gender: 'female' },
count: 2,
teachers: [
{
email: 'swara.desai@example.com',
age: 20,
name: 'swara desai',
gender: 'female'
},
{
email: 'mrunmayi.sawant@example.com',
age: 30,
name: 'mrunmayi sawant',
gender: 'female'
}
]
},
{
_id: { gender: 'male' },
count: 5,
teachers: [
{
email: 'sagar.desai@example.com',
age: 30,
name: 'sagar desai',
gender: 'male'
},
{
email: 'sagar.sawant@example.com',
age: 30,
name: 'sachin sawant',
gender: 'male'
},
{
email: 'sandesh.raorane@example.com',
age: 45,
name: 'sandesh raorane',
gender: 'male'
},
{
email: 'gaurav.raorane@example.com',
age: 42,
name: 'Gaurav raorane',
gender: 'male'
},
{
email: 'vinayak.tambe@example.com',
age: 21,
name: 'vinayak tambe',
gender: 'male'
}
]
},
{
_id: { gender: 'trans-gender' },
count: 1,
teachers: [
{
email: 'kiran.rao@example.com',
age: 21,
name: 'kiran rao',
gender: 'trans-gender'
}
]
}
]

2024/04/24

MongoDb Interview Question:Join multiple collections

Assume we have three collections (authors, authorInfo, and userRole) with the
following data:

db.authors.insertMany([
{
"_id" : ObjectId("5d0127aaee66e13d34c7c389"),
"address" : "32 Makram Ebeid Street",
"isActive" : true,
"authorId" : "121"
}
])

db.authorInfo.insertMany([
{
"_id" : ObjectId("5d0f726bac65f929d0fa98b2"),
"authorId" : "121",
"description" : "A description"
}
])


db.userRole.insertMany([
{
"_id" : ObjectId("5d012a08ee66e13d34c7c38f"),
"userId" : "121",
"role" : "manager"
}
])

How can we join the authors from all three collections

Answers:

db.authors.aggregate( [
{
$lookup:
{
from: "authorInfo",
localField: "authorId",
foreignField: "authorId",
as: "authorInfo"
}
},{
$unwind: "$authorInfo"
},
{
$lookup:
{
from: "userRole",
localField: "authorId",
foreignField: "userId",
as: "userRole"
}
},
{
$unwind:"$userRole"
},{
$project:{
_id:1,
address:1,
isActive:1,
authorId:1,
description:"$authorInfo.description",
role:"$userRole.role"
}
}
])

Note:
$lookup does not work in case of sharded collection.

MongoDb Interview Question:Filter Object Array

Problem:

Assume there is a collection named users that looks like the one below. How can
you get all houses in the “Rabia” neighborhood?

db.userHouses.insertMany([
{
"_id" : ObjectId("5d011c94ee66e13d34c7c388"),
"userName" : "kevin",
"email" : "kevin@toptal.com",
"password" : "affdsg342",
"houses" : [
{
"name" : "Big Villa",
"neighborhood" : "Zew Ine"
},
{
"name" : "Small Villa",
"neighborhood" : "Rabia"
}
]
},
{
"_id" : ObjectId("5d011c94ee66e13d34c7c387"),
"userName" : "sherif",
"email" : "sharief@toptal.com",
"password" : "67834783ujk",
"houses" : [
{
"name" : "New Mansion",
"neighborhood" : "Nasr City"
},
{
"name" : "Old Villa",
"neighborhood" : "Rabia"
}
]
},

])

Solution:

db.userHouses.find(
{"houses.neighborhood":"Rabia"},
{
"houses":{$elemMatch:{"neighborhood" : "Rabia"}},
"_id":0
}

)

or

db.userHouses.aggregate([
{ $match: { 'houses.neighborhood': 'Rabia' } },
{
$project: {
filteredHouses: { // This is just an alias
$filter: {
input: '$houses',
as: 'houseAlias',
cond: { $eq: ['$$houseAlias.neighborhood', 'Rabia'] }
}
},
_id: 0
}
}

])

MongoDb Interview Question:Insert into leaf node in nexted array

 Assume there is a document with nested arrays that looks like the one below. How

can you insert a “room” that has the name “Room 44” and size of “50” for a
particular “house” that belongs to this user?

db.users.insertOne({
"_id": "682263",
"userName" : "sherif",
"email": "sharief@aucegypt.edu",
"password": "67834783ujk",
"houses": [
{
"_id": "2178123",
"name": "New Mansion",
"rooms": [
{
"name": "4th bedroom",
"size": "12"
},
{
"name": "kitchen",
"size": "100"
}
]
}
]
})

Answer:
db.users.updateOne(
{ "_id": "682263","houses._id":"2178123"},
{ "$push":
{"houses.$.rooms":
{
"name": "Room 44",
"size": "50"
}
}
}
)

2024/04/22

MongoDb: $switch operator

We often used switch cases in javascript or C# or java as programming languages.

MongoDb also has switch case functionality.Lets explore switch case in mongodb.
It is about syntax only logic we already know.

$switch:
Evaluates a series of case expressions. When it finds an expression which
evaluates to true, $switch executes a specified expression and breaks out of the
control flow.

We will take a collection named grade as follows.

db.grades.insertMany([
{ "_id" : 1, "name" : "Susan Wilkes", "scores" : [ 87, 86, 78 ] },
{ "_id" : 2, "name" : "Bob Hanna", "scores" : [ 71, 64, 81 ] },
{ "_id" : 3, "name" : "James Torrelio", "scores" : [ 91, 84, 97 ] }
])

we will add Remarks based performance of student grade.if average score
is 90 or more we will give "Doing great!",If average score between 80
(inclusive) and 90 (exclusive) we will give "Doing pretty well."If average
score is less than 80 (exclusive) then will give Remarks "Needs improvement.".

db.grades.aggregate( [
{
$project:
{
"name" : 1,
"remarks" :
{
$switch:
{
branches: [
{
case: { $gte : [ { $avg : "$scores" }, 90 ] },
then: "Doing great!"
},
{
case: {
$and : [
{ $gte : [ { $avg : "$scores" }, 80 ] },
{ $lt : [ { $avg : "$scores" }, 90 ] }
]
},
then: "Doing pretty well."
},
{
case: { $lt : [ { $avg : "$scores" }, 80 ] },
then: "Needs improvement."
}
],
default: "No scores found."
}
}
}
}
] )

Output:
[
{ _id: 1, name: 'Susan Wilkes', remarks: 'Doing pretty well.' },
{ _id: 2, name: 'Bob Hanna', remarks: 'Needs improvement.' },
{ _id: 3, name: 'James Torrelio', remarks: 'Doing great!' }
]

Now we will re modify this grades document where scores will be
object array and rebuild this query

//drop collection if pre exists
db.alternateGrades.drop()

db.alternateGrades.insertMany([
{
_id : 1,
name : "Susan Wilkes",
scores: [
{subject:"math",score:87 },
{subject:"science",score:86 },
{subject:"social science",score:78 }
]
},
{
_id : 2,
name : "Bob Hanna",
scores : [
{subject:"math",score:67},
{subject:"science",score:93},
{subject:"social science",score:67}
]
},
{
_id : 3,
name : "James Torrelio",
scores : [
{subject:"math",score:91},
{subject:"science",score:84},
{subject:"social science",score:83}
]
}
])

Here is modified query

db.alternateGrades.aggregate( [
{
$project:
{
"name" : 1,
"averageScore": { $avg : "$scores.score" },
"remarks" :
{
$switch:
{
branches: [
{
case: { $gte : [ { $avg : "$scores.score" }, 90 ] },
then: "Doing great!"
},
{
case: {
$and : [
{ $gte : [ { $avg : "$scores.score" }, 80 ] },
{ $lt : [ { $avg : "$scores.score" }, 90 ] }
]
},
then: "Doing pretty well."
},
{
case: { $lt : [ { $avg : "$scores.score" }, 80 ] },
then: "Needs improvement."
}
],
default: "No scores found."
}
}
}
}
] )

Output:
[
{
_id: 1,
name: 'Susan Wilkes',
averageScore: 83.66666666666667,
remarks: 'Doing pretty well.'
},
{
_id: 2,
name: 'Bob Hanna',
averageScore: 75.66666666666667,
remarks: 'Needs improvement.'
},
{
_id: 3,
name: 'James Torrelio',
averageScore: 86,
remarks: 'Doing pretty well.'
}
]
The output is almost same just I added averageScore to verify if remarks are
given properly based on conditions decided.

Now suppose we decided to give remarks not based on average but based on individual
score in subject then,let us look into that query

db.alternateGrades.aggregate([
{
"$addFields": {
scores: {
$map: {
input: "$scores",
as: "score",
in: {
"$mergeObjects": [
"$$score",
{
"remark":{
$switch:
{
branches: [
{
case: { $gte : [ "$$score.score", 90 ] },
then: "Doing great!"
},
{
case: {
$and : [
{ $gte : [ "$$score.score", 80 ] },
{ $lt : [ "$$score.score", 90 ] }
]
},
then: "Doing pretty well."
},
{
case: { $lt : [ "$$score.score", 80 ] },
then: "Needs improvement."
}
],
default: "No scores found."
}
}
}
]
}
}
}
}
}
])

Output:
[
{
_id: 1,
name: 'Susan Wilkes',
scores: [
{ subject: 'math', score: 87, remark: 'Doing pretty well.' },
{ subject: 'science', score: 86, remark: 'Doing pretty well.' },
{
subject: 'social science',
score: 78,
remark: 'Needs improvement.'
}
]
},
{
_id: 2,
name: 'Bob Hanna',
scores: [
{ subject: 'math', score: 67, remark: 'Needs improvement.' },
{ subject: 'science', score: 93, remark: 'Doing great!' },
{
subject: 'social science',
score: 67,
remark: 'Needs improvement.'
}
]
},
{
_id: 3,
name: 'James Torrelio',
scores: [
{ subject: 'math', score: 91, remark: 'Doing great!' },
{ subject: 'science', score: 84, remark: 'Doing pretty well.' },
{
subject: 'social science',
score: 83,
remark: 'Doing pretty well.'
}
]
}
]

We can straight-way update our collection "alternateGrades" to hold
remarks as follows


db.alternateGrades.updateMany(
{},
[
{
$set: {
scores: {
$map: {
input: "$scores",
as: "score",
in: {
subject: "$$score.subject",
score: "$$score.score",
remark: {
$switch: {
branches: [
{
case: { $gte: [ "$$score.score", 90 ] },
then: "Doing great!"
},
{
case: {
$and: [
{ $gte: [ "$$score.score", 80 ] },
{ $lt: [ "$$score.score", 90 ] }
]
},
then: "Doing pretty well."
},
{
case: { $lt: [ "$$score.score", 80 ] },
then: "Needs improvement."
}
],
default: "No score found."
}
}
}
}
}
}
}
]
)

Here we are re-creating "scores" array where the lines

subject: "$$score.subject",
score: "$$score.score",

make sure that existing values remain intact and then it calculate "remark"
value using condition on score of that subject.

Lets once check how it is updated

db.alternateGrades.find({})

Output:
[
{
_id: 1,
name: 'Susan Wilkes',
scores: [
{ subject: 'math', score: 87, remark: 'Doing pretty well.' },
{ subject: 'science', score: 86, remark: 'Doing pretty well.' },
{
subject: 'social science',
score: 78,
remark: 'Needs improvement.'
}
]
},
{
_id: 2,
name: 'Bob Hanna',
scores: [
{ subject: 'math', score: 67, remark: 'Needs improvement.' },
{ subject: 'science', score: 93, remark: 'Doing great!' },
{
subject: 'social science',
score: 67,
remark: 'Needs improvement.'
}
]
},
{
_id: 3,
name: 'James Torrelio',
scores: [
{ subject: 'math', score: 91, remark: 'Doing great!' },
{ subject: 'science', score: 84, remark: 'Doing pretty well.' },
{
subject: 'social science',
score: 83,
remark: 'Doing pretty well.'
}
]
}
]

Here we can see that remark field is added in each element of scores
array based on conditions we put on score.

Now we will go further and in remark use subject name

Here is my query

db.alternateGrades.updateMany(
{},
[
{
$set: {
scores: {
$map: {
input: "$scores",
as: "score",
in: {
subject: "$$score.subject",
score: "$$score.score",
remark: {
$switch: {
branches: [
{
case: { $gte: [ "$$score.score", 90 ] },
then: {$concat: [`Doing great! in `,
"$$score.subject" ]}
},
{
case: {
$and: [
{ $gte: [ "$$score.score", 80 ] },
{ $lt: [ "$$score.score", 90 ] }
]
},
then: {$concat: [`Doing pretty well in `,
"$$score.subject" ]}
},
{
case: { $lt: [ "$$score.score", 80 ] },
then: {$concat: [`Needs improvement in `,
"$$score.subject" ]}
}
],
default: "No score found."
}
}
}
}
}
}
}
]
)

Lets once check once gain how it is updated

db.alternateGrades.find({})
Output:

[
{
_id: 1,
name: 'Susan Wilkes',
scores: [
{
subject: 'math',
score: 87,
remark: 'Doing pretty well in math'
},
{
subject: 'science',
score: 86,
remark: 'Doing pretty well in science'
},
{
subject: 'social science',
score: 78,
remark: 'Needs improvement in social science'
}
]
},
{
_id: 2,
name: 'Bob Hanna',
scores: [
{
subject: 'math',
score: 67,
remark: 'Needs improvement in math'
},
{
subject: 'science',
score: 93,
remark: 'Doing great! in science'
},
{
subject: 'social science',
score: 67,
remark: 'Needs improvement in social science'
}
]
},
{
_id: 3,
name: 'James Torrelio',
scores: [
{ subject: 'math', score: 91, remark: 'Doing great! in math' },
{
subject: 'science',
score: 84,
remark: 'Doing pretty well in science'
},
{
subject: 'social science',
score: 83,
remark: 'Doing pretty well in social science'
}
]
}
]