Tag: LoRaWAN

Storing your TTN node data in a noSQL database

This blog post is all about building a Node.js server to subscribe to the TTN MQTT(S) broker and store the data in a Mongo noSQL database. I am assuming that you have built a Node.js app before. There is also a suggestion of an application that will show your data.

The Things Network (TTN) is an open-source, secure and scalable solution that has around 5,000 gateways and 50,000 members around the world. It started in August 2015 and just keeps on growing.

TTN manages every aspect of creating and managing a LoRaWAN network except storing your data (note that TTN does offer a storage solution, but provides storage for 7 days only). There are several integrations available, including EVRYTHNG, AWS IoT, Cayenne and Tago. This example is useful if you want to keep your data in-house.

Just FYI, my example code, shown below, is running on a VPS with an instance of ubuntu xenial server, hosted by https://scaleway.com for around €3/month.

First, install the following components

Mosquitto – lightweight broker/client for MQTT
Node.js – javascript run-time environment – I used V8.x LTS
MongoDB – noSQL database, ideal for json-formatted data

Then create a Node.js application, which typically consists of

package.json – describes app, dependencies, licence, etc.
config.js – store for all configuration data – URL, username, etc
(Note: contains your db password & MQTT access key in plain text!)
server.js – the Node.js app
mqtt-ca.pem – enables use of TLS encrypted MQTT traffic (available from your TTN console)

package.json typical contents:

{
    "name": "ttn-mqtt-database",
    "description": "Listens to MQTT messages and stores in MongoDB",
    "version": "1.0.0",
    "dependencies": {
	"mongodb": "^3.0.3",
	"mqtt": "2.15.1",
	"ttn": "^2.3.1"
    },
    "repository": {
    	"type": "git",
      "url": "https://github.com/[your github repository].git"
    }
} 

config.js contains the configuration information to subscribe to your specific MQTT channel and the connection credentials and path for your mongo database

var config = {};

config.debug = process.env.DEBUG || true;

config.mqtt  = {};
config.mqtt.namespace = process.env.MQTT_NAMESPACE || '+/devices/+/up';
config.mqtt.protocol  = process.env.MQTT_PROTOCOL  || 'mqtts://';
config.mqtt.hostname  = process.env.MQTT_HOSTNAME  || 'eu.thethings.network';
config.mqtt.port      = process.env.MQTT_PORT      || 8883;
config.mqtt.user      = process.env.MQTT_USER      || '[name of your TTN application]';
config.mqtt.password  = process.env.MQTT_PASSWORD  || 'ttn-account-v2.[your TTN key]';
config.mqtt.cafile    = process.env.MQTT_CAFILE    || 'mqtt-ca.pem';

config.mongodb = {};
config.mongodb.user       = process.env.MONGODB_USER       || '[user]';
config.mongodb.password   = process.env.MONGODB_PASSWORD   || ‘[your ultra-secret password]';
config.mongodb.hostname   = process.env.MONGODB_HOSTNAME   || '[your server DNS]';
config.mongodb.port       = process.env.MONGODB_PORT       || 27017;
config.mongodb.database   = process.env.MONGODB_DATABASE   || 'mqtt';
config.mongodb.collection = process.env.MONGODB_COLLECTION || 'nodedata';

module.exports = config;

The code for your Node.js server should look something similar to this:

server.js

var mongodb  = require('mongodb');
var mqtt     = require('mqtt');

var fs       = require('fs');
var config   = require('./config');

var client = mqtt.connect(config.mqtt.hostname, {
            ca: [fs.readFileSync(config.mqtt.cafile)],
          username: config.mqtt.user,
          password: config.mqtt.password,
              port: config.mqtt.port
         });

client.on('connect', function () {
    client.subscribe(config.mqtt.namespace);
});

var mongoUri = 'mongodb://' + config.mongodb.user + ':' + config.mongodb.password + '@' + config.mongodb.hostname + ':' + config.mongodb.port + '/' + config.mongodb.database;

console.log(mongoUri);
mongodb.MongoClient.connect(mongoUri, function(err, client) {
    if(err != null) {
        console.log(mongoUri);
        throw error;
    }
    else {
        var mydb = client.db('mqtt');
        var collection = mydb.collection(config.mongodb.collection);
        collection.createIndex({ "topic" : 1 });
        client.on('message', function (topic, message) {
        var messageObject = {
            topic: topic,
            message: message.toString()
        };
        console.log(message.toString());
        collection.insert(messageObject, function(err, result) {
            if(err != null) {
                console.log("ERROR: " + err);
            }
        });
    });
});

To see the data stored in your mongo database, either use the mongo client:

cli access via mongoDB client
user@demo:~/ttn-demo# mongo
> use mqtt
> db.message.find();

Or use a GUI app, like NoSQLBooster for MongoDB

https://nosqlbooster.com/downloads

A useful noSQL query would be similar to this, which shows the last 20 messages from a specific device called pycomr001:

db.nodedata.where({'device': 'pycomr001'}).sort({ "natural": -1}).limit(20);

So, in summary:
TTN provides an excellent platform to move your node data to the cloud
TTN enables access via a secure MQTT broker
MongoDB is a noSQL db, good for JSON data
Node.js is a simple way to build a headless server app
NoSQLBooster for MongoDB is a good GUI to access your db
systemctl is a sensible method to manage your app

Now, it’s down to you to do something useful with your data!

Technical overview of LoRa, LoRaWAN and The Things Network

The Radio Spectrum

Radio-connected devices work within a certain frequency range. The radio spectrum is allocated by government and licensed for specific functions.

For example, broadcast radio currently has a reserved VHF band from 88 to 108 MHz.

Cellular phones have several reserved frequency bands around 900 MHz and 1800 MHz.

The licensees pay for use within these bands. However, there are some specific bands that are made available for low power use, license-free.

2.4GHz is a global band that allows low power unlicensed operation; this is where WiFi and Bluetooth co-exist.

Some license-free bands are allocated differently by each country and LoRa is designed to work in the appropriate band for each country or region. For example, Europe is covered by 863-870MHz, North America, Canada & South America by 902-928MHz. TTN provide details of global frequency plans – https://www.thethingsnetwork.org/docs/lorawan/frequency-plans.html

LoRa radio technology

LoRa is a technology that works well within a crowded radio spectrum.

It uses a technique called chirp spread spectrum that creates a signal which is distinguishable by the receiver from the atmospheric noise and signals created by other devices that use simpler modulation techniques, such as On Off Keying (OOK) and Frequency Shift Keying (FSK).

LoRa also takes advantage of a feature called Spreading Factor (SF) where the spreading factors are the duration of the chirp. LoRa operates with SF7-12. SF7 is the shortest time on air, SF12 the longest. Each step up in spreading factor doubles the time on air to transmit the same amount of data.

With the same bandwidth, longer time on air obviously results in less data transmitted per unit of time.

So, with very little output power, the LoRa signal can be detected at great distance or through infrastructure that attenuates the signal. Typically, this could be 1-3km in an urban environment. The current record distance is just over 700km from a node under a helium balloon at a height of 40km, at an output power of only 25mw. For comparison your car key fob works at 2mw for a couple of metres and your phone at up to 1W to the nearest cellphone mast, at most 30km distant.

The trade-off for LoRa is that low power and long distance is traded for limited data rate. Maximum data rate at SF7 is 6kbits/s, reducing to 300bits/s at SF12. One key figure RF engineers discuss is “link budget”, bigger is better and Spreading Factor affects the number where SF12 gives the best chance to receive data. For more information – https://www.semtech.com/uploads/documents/an1200.22.pdf

There are payload size limitations for LoRa – in EU it’s 230 bytes per transmission at SF7, 59 bytes at SF12. Consider LoRa only for telemetry data, such as location, heading, height, temperature, pressure and humidity values or simple on-off actions.

The LoRaWAN protocol

LoRaWAN is a communication protocol that runs on LoRa hardware.

LoRaWAN operates in a star of stars network with ability for multiple gateways to receive and forward data from any nearby node to The Things Network.

Each multi-channel LoRaWAN gateway can scan 8 channels simultaneously and decode up to 8 data packets at the same time. Several packets using different data rates may be demodulated simultaneously even on the same channel.

LoRaWAN provides several modes of operation – modes A, B & C, where B is allocated a time slot and C devices are permanently powered on and receiving.

Most nodes run as mode A devices where they occasionally wake up to transmit a small amount of data and listen for a short time after transmission for any received data. Working like this, it is possible to construct a node that can transmit an update every 30 minutes and run off 2 ‘C’-size alkaline batteries for more than 5 years.

Each mode A node is free to transmit at any time, with the limitation in Europe that each node should not exceed a 1% duty cycle.

There are predictive formulae available to show that the combination of multi-channel gateways, small data packets, operational modes and Spreading Factors results in the capability to handle thousands of nodes per gateway. See “Understanding the Limits of LoRaWAN (Adelantado, Vilajosana et al, IEEE magazine January 2017)” – https://arxiv.org/pdf/1607.08011.pdf

The Things Network originally provided complete LoRaWAN coverage for the city of Amsterdam with just 10 gateways – https://www.thethingsnetwork.org/community/amsterdam/

The Things Network

The Things Network is an open-source, free-to-use solution that provides all functions necessary to:

  • Register a gateway
  • Register a node
  • Handle encryption and decryption of payload data.
  • Route the data between node and the internet using MQTT-broker technology.
  • Queue data for transmission to each node.
  • Monitor and action the process through API

The Things Network does all this and more on a scalable, resilient platform with regional bridge access globally.

https://www.thethingsnetwork.org/docs/network/architecture.html

The Things Network is provided as open-source solution by The Things Industries. Gateways are being made available worldwide by The Things Network community – see https://www.thethingsnetwork.org/community and try it for yourself! it costs less than £200 to setup your own.

The Things Industries also deliver private LoRaWAN solutions with guaranteed SLA, so be assured that you can develop a solution and build it out to a stable product.

For more information, please get in touch – take a look at https://thinnovation.com/remon.html

STMicro DISCO-L072CZ-LRWAN1 quickstart with mbed-os

There is a great article on the ARM mbed website about their support of LoRaWAN.
https://os.mbed.com/blog/entry/Adding-a-LoRaWAN-stack-to-Mbed-OS-58/

Their code has improved a lot this year. Previously, it was a little tricky to work with – there is now a clean install available on github that can be built locally from the command line on Mac, Windows or linux. using the GNU ARM embedded toolchain and mbed-os.

I describe below a simple recipe to use this code with the STMicro DISCO-L072CZ-LRWAN1 evaluation board (pictured). The code can be simply adapted to run with the sx1272 mbed shield on STMicro Nucleo boards and there is also support for the RAKWireless RAK811 node.

I am assuming that you have already registered your device in your application in your TTN console:
https://console.thethingsnetwork.org/applications/[your application name]/devices

1. Setup the GNU ARM embedded toolchain and install mbed-cli:

On a Windows 10 PC – just use the installer that you can download from here:
mbed-cli installer

The Windows installer for Mbed CLI includes the following components:

  • Python – Mbed CLI is a Python script, so you need Python to use it. The installer installs version 2.7.13 of Python. It is not compatible with Python 3
  • Mbed CLI version 1.2.2 – Mbed CLI
  • Git and Mercurial – Mbed CLI supports both Git and Mercurial repositories. Both Git and Mercurial are being installed. git and hg are added to system’s PATH
  • GNU Arm Embedded Toolchain – GNU Embedded Toolchain for Arm
  • Mbed Windows serial port driver – serial port driver

Note that the Windows installer sets all paths and even configures some MBED variables
Check the path to the gcc compiler:
mbed config -L

On a Mac – use homebrew – remember to brew update && brew upgrade before installing
If you need to install homebrew first, then
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
Otherwise, just:

brew tap ArmMbed/homebrew-formulae
brew install arm-none-eabi-gcc
brew install python@2
brew install mercurial
brew install git
git clone https://github.com/ARMmbed/mbed-cli
cd mbed-cli
python setup.py install

Check the version of mbed-cli and that it is installed OK – version should be =>1.8.2
mbed-cli --version
Check the install location of arm-none-eabi-gcc with
which arm-none-eabi-gcc
Set path for mbed
mbed config -G GCC_ARM_PATH "/usr/local/bin"

On a linux machine, here is a typical install (I used ubuntu xenial)

sudo apt-get install software-properties-common
sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa
sudo apt-get update
sudo apt-get install gcc-arm-embedded
sudo apt-get install build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev
sudo apt-get install mercurial git python-dev python-pip
pip install mbed-cli
mbed-cli --version
which arm-none-eabi-gcc
mbed config -G GCC_ARM_PATH "/usr/bin"

2. Pull the example code from the mbed github and create the local mbed-os environment in the folder, using mbed-cli (note that ‘mbed-cli’ also works using just ‘mbed’)

mbed import mbed-os-example-lorawan
cd mbed-os-example-lorawan/

3. Configure for your node
edit mbed_app.json to configure lora.device-eui, lora.application-eui, lora.application-key

4. compile and send to evaluation board (assumes there is only one ST-Link/Nucleo/DISCO board connected)
mbed compile -m DISCO_L072CZ_LRWAN1 -t GCC_ARM --flash

5. use mbed as your terminal emulator:
mbed sterm --baudrate 115200

Note that on linux, it may be necessary to change user permissions for the serial port, similar to:
sudo chmod 666 /dev/ttyACM0

6. press the black reset button on the DISCO board
You should see this:

 Mbed LoRaWANStack initialized
 CONFIRMED message retries : 3
 Adaptive data  rate (ADR) - Enabled
 Connection - In Progress ...

After a few seconds, this should appear:

 Connection - Successful
 Dummy Sensor Value = 2.1
 25 bytes scheduled for transmission
 Message Sent to Network Server

Any questions? Please get in touch…

Using node-red with ST NUCLEO L073RZ Sx1272

I have been working with the great LoRaWAN combo from STMicroelectronics – a low-cost dev board based on a ARM®32-bit Cortex®-M0+ CPU, coupled with the Sx1272 on an Arduino-compatible shield.

Using the online mbed compiler – https://developer.mbed.org/compiler/ , I modded the demo code – LoRaWAN-demo-72 – to add simple output to an OLED display – see attached photo.

My node-red install runs on a server hosted by https://www.scaleway.com/ that costs around €3/month. I designed a very simple activity monitor – the design of which may help others.

To the basic node-red install, I added node-red-contrib-ttn and node-red-dashboard.

file

The dashboard output looks like this:

file

Here is the node-red code:

[{"id":"cd89d617.a9d0e8","type":"function","z":"546def68.3e374","name":"Extract data","func":"count = {}\nrssi = {}\nsnr = {}\n\n//var b = new Buffer(msg.payload_raw);\n//count.payload = (b[1] << 8 | b[2]).toString();\n//var r = b[3] << 8 | b[4];\n//if ((r & 0x8000) > 0) {\n// r = r - 0x10000;\n//}\n//rssi.payload = r.toString();\n//snr.payload = (b[5]).toString();\n\ncount.payload = msg.counter;\n\nrssi.payload = msg.metadata.gateways[0].rssi;\nsnr.payload = msg.metadata.gateways[0].snr;\n\nreturn [count, rssi, snr];","outputs":"3","noerr":0,"x":230,"y":280,"wires":[["6de8dcb1.29b284"],["72787db7.d4bdf4","a262c4ea.068798"],["21f1441a.e028fc","e6796eca.3e2ec"]]},{"id":"6de8dcb1.29b284","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":7,"width":0,"height":0,"name":"txtCount","label":"Count","format":"{{msg.payload}}","layout":"row-spread","x":440,"y":220,"wires":[]},{"id":"72787db7.d4bdf4","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":5,"width":0,"height":0,"name":"txtRssi","label":"RSSI","format":"{{msg.payload}}","layout":"row-spread","x":430,"y":260,"wires":[]},{"id":"21f1441a.e028fc","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":6,"width":0,"height":0,"name":"txtSnr","label":"SNR","format":"{{msg.payload}}","layout":"row-spread","x":430,"y":340,"wires":[]},{"id":"69c26716.6f6598","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":1,"width":0,"height":0,"name":"txtDevice","label":"Device","format":"{{msg.dev_id}}","layout":"row-spread","x":440,"y":60,"wires":[]},{"id":"89e50c80.55a11","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":3,"width":0,"height":0,"name":"txtDataRate","label":"Data Rate","format":"{{msg.metadata.data_rate}}","layout":"row-spread","x":450,"y":140,"wires":[]},{"id":"d312c0a6.afb48","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":4,"width":0,"height":0,"name":"txtCodingRate","label":"Coding Rate","format":"{{msg.metadata.coding_rate}}","layout":"row-spread","x":460,"y":180,"wires":[]},{"id":"2065149f.497fbc","type":"ui_text","z":"546def68.3e374","group":"3d90cd90.e1c9c2","order":2,"width":0,"height":0,"name":"txtFrequency","label":"Freq.","format":"{{msg.metadata.frequency}}","layout":"row-spread","x":450,"y":100,"wires":[]},{"id":"a262c4ea.068798","type":"ui_chart","z":"546def68.3e374","name":"","group":"3d90cd90.e1c9c2","order":8,"width":0,"height":0,"label":"Signal Strength","chartType":"line","legend":"false","xformat":"HH:mm:ss","interpolate":"linear","nodata":"","dot":false,"ymin":"-150","ymax":"0","removeOlder":1,"removeOlderPoints":"","removeOlderUnit":"3600","cutout":0,"useOneColor":false,"colors":["#1f77b4","#aec7e8","#ff7f0e","#2ca02c","#98df8a","#d62728","#ff9896","#9467bd","#c5b0d5"],"useOldStyle":true,"x":460,"y":300,"wires":[[],[]]},{"id":"e6796eca.3e2ec","type":"ui_chart","z":"546def68.3e374","name":"SNR","group":"3d90cd90.e1c9c2","order":9,"width":0,"height":0,"label":"Signal:Noise Ratio","chartType":"line","legend":"false","xformat":"HH:mm:ss","interpolate":"linear","nodata":"","dot":false,"ymin":"0","ymax":"10","removeOlder":1,"removeOlderPoints":"500","removeOlderUnit":"3600","cutout":0,"useOneColor":false,"colors":["#1f77b4","#aec7e8","#ff7f0e","#2ca02c","#98df8a","#d62728","#ff9896","#9467bd","#c5b0d5"],"useOldStyle":true,"x":430,"y":380,"wires":[[],[]]},{"id":"7819848e.fd117c","type":"ttn uplink","z":"546def68.3e374","name":"unosx1272","app":"c7f29eb8.f2f1f","dev_id":"unosx1272","field":"","x":120,"y":60,"wires":[["cd89d617.a9d0e8","69c26716.6f6598","2065149f.497fbc","89e50c80.55a11","d312c0a6.afb48"]]},{"id":"3d90cd90.e1c9c2","type":"ui_group","z":"546def68.3e374","name":"unosx1272","tab":"a104b487.87dca8","order":2,"disp":true,"width":"6","collapse":false},{"id":"c7f29eb8.f2f1f","type":"ttn app","z":"","appId":"<app-id>","accessKey":"<app-key>","discovery":"discovery.thethingsnetwork.org:1900"},{"id":"a104b487.87dca8","type":"ui_tab","z":"546def68.3e374","name":"devices","icon":"dashboard"}]