Search This Blog

2024/05/17

Node.js:Getting latitude & longitude based on address.

Recently I was got into situation where I have tables of country,state,
city with

proper foreign key and referenced in other data but I need latitude &
longitude for city.I does not wanted to create new table so I altered City
Table.


My City Altered Table schema in MySQL after modification look like below

CREATE TABLE `City` (
`id` int unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL,
`countryId` int unsigned NOT NULL,
`stateId` int unsigned NOT NULL,
`createdAt` datetime NOT NULL,
`updatedAt` datetime NOT NULL,
`deletedAt` datetime DEFAULT NULL,
`latitude` float DEFAULT NULL,
`longitude` float DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `countryId` (`countryId`),
KEY `stateId` (`stateId`),
CONSTRAINT `City_ibfk_5` FOREIGN KEY (`countryId`) REFERENCES `Country` (`id`),
CONSTRAINT `City_ibfk_6` FOREIGN KEY (`stateId`) REFERENCES `State` (`id`)
) ENGINE=InnoDB


Now I left with way to find latitude & longitude for each city ,some data is
available freely online but it is needed to inserted into new table then extract
latitude & longitude for my table.

I came across a API provider by geocode.maps.co that lets you get latitude &
longitude from address. I registered for API Key ,its free plan lets us make
100,000 api calls per month with limit 1 request per second.

Below is code in Node.js to accomplish same.

What I am doing in code.

1) Getting City which does not have latitude & longitude in City table along
with country & state names
2) Making API call to get latitude & longitude based on city name, state
name,country name.
3) Update the City record with its latitude & longitude.

NPM dependencies used in code:
npm i mysql2

Node.js Code:

const mysql = require('mysql2/promise');
const apiKey = '{Your API Key}';

// Function to create a delay
function delay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}

async function getCityWithoutLatLong() {
let connection;
try {
connection = await getConnection();
const query = 'SELECT country.id as countryId, country.name
                countryName, state.id as stateId, state.name stateName,
                city.id as cityId, city.name cityName FROM City city
                INNER JOIN Country country ON city.countryId = country.id
                INNER JOIN State state ON city.stateId = state.id
                WHERE city.countryId = 101;';
const [results] = await connection.query(query);

// Process each city sequentially with a delay
for (const element of results) {
const address = `${element.countryName},${element.stateName},
${element.cityName}`;
const apiEndpoint = `https://geocode.maps.co/search?
q=${address}&api_key=${apiKey}`;
console.log(`Calling API with cityId=${element.cityId}`);
await callAPI(element.cityId, apiEndpoint);
await delay(1000); // Wait for 1 second before the next API call
}
} catch (err) {
console.error('Error in getCityWithoutLatLong:', err);
} finally {
if (connection && connection.end) await connection.end();
console.log("Exited Process in function getCityWithoutLatLong");
process.exit();
}
}

// Function to call an API and process the response
async function callAPI(cityId, apiEndpoint) {
try {
const requestOptions = {
method: "GET",
redirect: "follow"
};

const response = await fetch(apiEndpoint, requestOptions);

if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}

const data = await response.json();

const lat = data[0].lat;
const long = data[0].lon;

console.log(`lat=${lat}, long=${long}`);

const updateResult = await updateCityWithLatLong(cityId, lat, long);
console.log("City updated:", updateResult);
} catch (error) {
console.error('Fetch error:', error);
}
}

// Function to update the city with latitude and longitude in the database
async function updateCityWithLatLong(cityId, lat, long) {
let connection;
try {
connection = await getConnection();
const query = `UPDATE City SET latitude = ?, longitude = ? WHERE id = ?`;
const [results] = await connection.query(query, [lat, long, cityId]);
return results;
} catch (err) {
console.error('Database query error:', err);
} finally {
if (connection && connection.end) await connection.end();
}
}

// Function to get a connection to the database
async function getConnection() {
const connection = await mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'sangram#81',
database: 'MatrimonyDb'
});
return connection;
}

getCityWithoutLatLong();


In My Code I Hardcoded Id of Country India i.e. 101.Modify My Code as per your
table structure,You need to replace API KEY in above code.

2024/05/04

Dynamoose & Express.js :CRUD Operations

Today we explore dynamoose a ODM or dynamodb.I am using local dynamodb
instead of cloud instance.

As usual I am creating by base express template using generator.

express --view=ejs dynamoose-express

Now i will create a demo.js in route & mount it in app.js at mountpoint 'demo'

Content of demo.js

var express = require("express");
var router = express.Router();
var User = require("../models/userModel");

/* GET home page. */
router.post("/createUser", async function (req, res, next) {
try {
// Use the model
const user = new User({
name: req.body.name,
email: req.body.email,
age: req.body.age,
});

// Save the user to DynamoDB
let savedUser = await user.save({ overwrite: false });
savedUser.createdAt = new Date(savedUser.createdAt)

res.json({
success: true,
message: "User created Successully",
data: savedUser,
});
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

//get record by email
router.get("/getByEmail/:email", async function (req, res, next) {
try {
let user = await User.query("email").eq(req.params.email).exec();
res.json({ success: true, message: "User found", data: user });
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

//get record by user id
router.get("/getByUserId/:userId", async function (req, res, next) {
try {
let user = await User.query("userId").eq(req.params.userId).exec();
res.json({ success: true, message: "User found", data: user });
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

//get all records
router.get("/getAll", async function (req, res, next) {
try {
const users = await User.scan().exec();
res.json({ success: true, message: "User found", data: users });
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

//greater than
router.get("/youngerThan/:age", async function (req, res, next) {
try {
const users = await User.scan()
.filter("age")
.gt(parseInt(req.params.age))
.exec();
res.json({ success: true, message: "User found Successully", data: users });
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

//delete
router.delete("/:userId", async (req, res, next) => {
try {
let myUser = await User.query("userId").eq(req.params.userId).exec();
if (myUser.length) {
await myUser[0].delete(); // Delete the user with the specified userId
res.json({ success: true, message: "User deleted successfully" });
} else {
res.json({ success: false, message: "User not found" });
}
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

//update
router.put("/:userId", async (req, res, next) => {
try {
let myUser = new User({
name: req.body.name,
email: req.body.email,
age: req.body.age,
userId: req.params.userId,
updatedAt: Date.now(),
});

myUser = await myUser.save({ overwrite: true });

myUser.updatedAt = new Date(myUser.updatedAt)
myUser.createdAt = new Date(myUser.createdAt)

res.json({
success: true,
message: "User saved successfully",
data: myUser,
});
} catch (exp) {
res.json({ success: false, message: exp.message });
}
});

module.exports = router;

Changes required in app.js

var demoRouter = require('./routes/demo');
&
app.use('/demo', demoRouter);

Now lets install required npm packages

Please check the packages to install from package.json below

{
"name": "dynamoose-express",
"version": "0.0.0",
"private": true,
"scripts": {
"start": "node ./bin/www"
},
"dependencies": {
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"dotenv": "^16.4.5",
"dynamoose": "^4.0.1",
"ejs": "~2.6.1",
"express": "^4.19.2",
"http-errors": "~1.6.3",
"morgan": "~1.9.1",
"uuid": "^9.0.1"
}
}

Now add connection.js at root

const dynamoose = require('dynamoose');

dynamoose.aws.ddb.local("http://localhost:8000")
module.exports = dynamoose


create a models folder and add userModel.js into it as

const dynamoose = require("../connection");
const { v4: uuidv4 } = require('uuid');
const UserSchema = new dynamoose.Schema(
{
userId: {
type: String,
hashKey: true,
default:uuidv4()
},
email: {
type: String,
index: {
name: "EmailIndex",
global: true,
rangeKey: "userId",
},
},
name: {
type: String,
},
age: Number,
createdAt: {
type: Date,
default: Date.now,
},
updatedAt: {
type: Date,
default: null,
},
},
{
throughput: "ON_DEMAND", // or { read: 5, write: 5 }
}
);

const User = dynamoose.model("User", UserSchema);
module.exports = User;

My .env file is as below

AWS_ACCESS_KEY_ID = "fakeMyKeyId"
AWS_SECRET_ACCESS_KEY = "fakeSecretAccessKey"
AWS_REGION = "fakeRegion"

You can run npm install meanwhile if npm packages not yet installed.

Now to check & rest endpoint created run npm start.

Endpoint:

For creation of User:

curl --location 'http://localhost:3000/demo/createUser' \
--header 'Content-Type: application/json' \
--data-raw '{
"name":"Vijay Desai",
"email":"Vijay@gmail.com",
"age":24
}'

Output:
{
"success": true,
"message": "User created Successully",
"data": {
"name": "Vijay Desai",
"email": "Vijay@gmail.com",
"age": 24,
"userId": "fda3f7e2-c835-45ff-853c-b93f8b26cb93",
"createdAt": "2024-05-04T13:28:45.554Z"
}
}

For Updation of Existing User.

curl --location --request PUT 'http://localhost:3000/demo/2' \
--header 'Content-Type: application/json' \
--data-raw '{
"name": "Sangram Desai",
"email": "sangram@gmail.com",
"age": 43
}'

Output:
{
"success": true,
"message": "User saved successfully",
"data": {
"name": "Sangram Desai",
"email": "sangram@gmail.com",
"age": 43,
"userId": "2",
"updatedAt": "2024-05-04T13:31:30.480Z",
"createdAt": "2024-05-04T13:31:30.483Z"
}
}

For viewing list o all users:
curl --location 'http://localhost:3000/demo/getAll'

For getting user by its userId:

curl --location 'http://localhost:3000/demo/getByUserId/fda3f7e2-c835-45ff-853c-b93f8b26cb93'

There are some other endpoint that you can explore.

The complete code of this project is available at
https://github.com/gitsangramdesai/dynamoose-express.

In DynamoDb there is no inbuild functionlity that is similar to auto-increment id in mysql,
people usually use uuid's for primary key.primary key in Dynamodb is made up of partition key
& sort key.Records with same partition key will be saved in one partition & sorted by
sort key(range key).You can also define primary key without sort key(range key)

AWS DynamoDB: How to install locally in Ubuntu?

It is possible to install dynamodb locally,lets explore how.

First Download jar file

wget https://d1ni2b6xgvw0s0.cloudfront.net/v2.x/dynamodb_local_latest.tar.gz

Now extract the the archieve.

Rename extracted folder as dynamodb

Copy it to location where you want to install binary

mv dynamodb /usr/share/

Now on command line go to folder in which dynamodb is copied,
in this case /usr/share/dynamodb.

cd /usr/share/dynamodb

Test i it runs from terminal by running this

sudo java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb

Now if you want to test the server each time you would have to run this command
To keep it running each time without running manually we need a service.

To create a service

sudo nano /etc/systemd/system/dynamodb.service

add ollowing to it

[Unit]
Description=DynamoDB Service
[Service]
User=root
WorkingDirectory=/usr/share/dynamodb
ExecStart=/usr/share/dynamodb/dynamodb.sh
SuccessExitStatus=143
TimeoutStopSec=10
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target

save & exit


Now we will create a shell script file referred in service above.

cd /usr/share/dynamodb
nano dynamodb.sh

add

#!/bin/sh
sudo java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb


save and exit.

Make shell script executable file

chmod u+x dynamodb.sh

Now let system know that we have created a new service

sudo systemctl daemon-reload
sudo systemctl enable dynamodb
sudo systemctl start dynamodb
sudo systemctl status dynamodb


output of last command in my case is like below

● dynamodb.service - Dynamo DB Local Service
Loaded: loaded (/etc/systemd/system/dynamodb.service; enabled; vendor preset: enabled)
Active: active (running) since Sat 2024-05-04 11:13:52 IST; 11min ago
Main PID: 33499 (dynamodb.sh)
Tasks: 41 (limit: 18708)
Memory: 150.8M
CPU: 4.333s
CGroup: /system.slice/dynamodb.service
├─33499 /bin/sh /usr/share/dynamodb/dynamodb.sh
├─33500 sudo java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb
└─33501 java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb

May 04 11:13:52 sangram-Inspiron-14-5430 sudo[33500]: root : PWD=/usr/share/dynamodb ; USER=root ; COMMAND=/usr/bin/java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb
May 04 11:13:52 sangram-Inspiron-14-5430 sudo[33500]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=0)
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: Initializing DynamoDB Local with the following configuration:
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: Port: 8000
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: InMemory: false
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: Version: 2.4.0
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: DbPath: null
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: SharedDb: true
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: shouldDelayTransientStatuses: false
May 04 11:13:53 sangram-Inspiron-14-5430 dynamodb.sh[33501]: CorsParams: null


Now we need to install awscli on ubuntu as ollows

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"
-o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install


Now run aws configure
and give below response to when prompted.

AWS Access Key ID [****************yId"]: fakeMyKeyId
AWS Secret Access Key [****************Key"]: fakeSecretAccessKey
Default region name ["fakeRegion"]: fakeRegion
Default output format [None]:


The response need to be given as given above.


Now you can check lists o tables in dynamodb by running follwoing command

aws dynamodb list-tables --endpoint-url http://localhost:8000

You may also like to check how to do CRUD oprtion in dynamoDb using exprress
https://msdotnetbuddy.blogspot.com/2023/05/working-with-dynamo-db.html.

References:
https://medium.com/aws-lambda-serverless-developer-guide-with-hands/
amazon-dynamodb-primary-key-partition-key-and-sort-key-how-to-choose-right-
key-for-dynamodb-ea5673cb87c0