Introduction: VR Vision
VR Vision bot was showcased in TechEvince 5.0 , IIT Guwahati , as a simple image processing based human-machine interface with a picam fitted in it through which we can see live reality on oculus .Its utility lies in the fact that no additional sensors or wearable except a glove is required to control the bot that runs on differential drive principle. In this instructable, we will take you through the working principle behind object tracking , gesture detection and live video transfer and detection used in the system. The source code of this project can be downloaded from Github via link: https://github.com/shwetank6044/gesture-hawk.git
Virtual reality is used to describe 3D, computer
generated environment which can be explored and interacted by a person . In this project , we have used the VR to show the reality which we can’t see by just sitting in our room , through VR vision robot and hand gestures . The person using the VR headset is able to interact with the robot through hand gestures and head movement and see the reality outside through the pi cam fitted on it without actually going out .
Step 1: Items Required
1)L298N Motor Driver
2)DC Motors
3)Robot car chassis
4)Arduino Uno
5)LiPo Batteries
6)Arduino USB Cable(long)
7)Servo (2) - one 180 degree and one 360 degree
8)Rasberry pi
9)oculus
10)Jumper wires
Step 2: Working Principle
Step 3: Working
The PiCam feed from the robot is hosted on a local Wi-Fi
network using a Raspberry Pi. An MPU6050 is used to track the head movement, which accordingly controls the PiCam placed on the servo mount so that the feed can be received from all directions. The feed is then received and displayed on a VR Headset. Another MPU6050 module is used to read the angles. An RF Transmitter sends the data to the Arduino on the bot. The data is processed and the bot is controlled accordingly.
Link for working of gesture car -
www.instructables.com/id/Gesture-Control-Car-Using... Mpu6050-and-Arduino/
Link for video transfer working -
https://www.youtube.com/watch?v=JjPsW-7FUng
Step 4: CODES
All the respective connections are given in the Code .
NRF TRANSMITTER
#include
#include
#include
#include
#include
"RF24.h" //SCK -> 13//MISO -> 12//MOSI -> 11//CSN -> 7//CE ->8
int msg[1];
RF24 radio(8,7);
const uint64_t pipe = 0xE8E8F0F0E1LL;
int potpin = 0;
int val;
MPU9250 mpu;
// void setup() { radio.begin(); Serial.begin(115200); radio.openWritingPipe(pipe);
//Sending the Data } void loop() { while(Serial.available()) { Vector normaccel = mpu.readNormalizeAccel();
int pitch = -(atan2(normAccel.XAxis, sqrt(normAccel.YAxis*normAccel.YAxis + normAccel.ZAxis*normAccel.ZAxis))*180.0)/M_PI;
int roll = (atan2(normAccel.YAxis, normAccel.ZAxis)*180.0)/M_PI;
Serial.print(" \nPitch = ");
Serial.print(pitch);
Serial.print(" Roll = ");
Serial.print(roll);
if(pitch > 20) { msg[0] = 1;
Serial.println(msg[0]);
radio.write(msg, 1);
delay(1000);
} else if(pitch < -20) { msg[0] = 2;
Serial.println(msg[0]);
radio.write(msg, 1);
delay(1000);
} else { msg[0] = 0;
Serial.println(msg[0]);
radio.write(msg, 1);
delay(1000);
}
if(roll > 20) { msg[0] = 3;
Serial.println(msg[0]);
radio.write(msg, 1);
delay(1000); }
else if(roll < -20) { msg[0] = 4;
Serial.println(msg[0]);
radio.write(msg, 1);
delay(1000); }
else { msg[0] = 0;
Serial.println(msg[0]);
radio.write(msg, 1);
delay(1000); }
}
}
NRF RECIEVER
//connection for rf receiver
//GND = GND
//DATA = 12
//VCC = 5V
#include
//connection for motors
//motor one int enA = 10;
int in1 = 9; int in2 = 8;
// motor two int enB = 11;
int in3 = 7;
int in4 = 6;
void setup() { vw_set_ptt_inverted(true);
// Required for DR3100 vw_set_rx_pin(12);
vw_setup(4000);
// Bits per sec
// set all the motor control pins to outputs
pinMode(enA, OUTPUT);
pinMode(enB, OUTPUT);
pinMode(in1, OUTPUT);
pinMode(in2, OUTPUT);
pinMode(in3, OUTPUT);
pinMode(in4, OUTPUT);
Serial.begin(9600); vw_rx_start();
// Start the receiver PLL running
}
void loop()
{
uint8_t buf[VW_MAX_MESSAGE_LEN];
uint8_t buflen = 2;
if (vw_get_message(buf, &buflen))
// Non-blocking
{
Serial.print(buf[0]) ;
Serial.println(buf[1]);
if(buf[0]=='f'){ Serial.print("forward");
Serial.println(buf[1]);
digitalWrite(in1, LOW);
digitalWrite(in2, HIGH);
analogWrite(enA,255);
digitalWrite(in3, HIGH);
digitalWrite(in4, LOW);
analogWrite(enB,255);
}
if(buf[0]=='b') { Serial.print("backward");
Serial.println(buf[1]);
digitalWrite(in1, HIGH);
digitalWrite(in2, LOW);
analogWrite(enA,255);
digitalWrite(in3, LOW);
digitalWrite(in4, HIGH);
analogWrite(enB,255);
}
if(buf[0]=='s'){ Serial.print("stop");
Serial.println(buf[1]);
digitalWrite(in1, LOW);
digitalWrite(in2, LOW);
analogWrite(enA,255);
digitalWrite(in3, LOW);
digitalWrite(in4, LOW);
analogWrite(enB,255);
}
if(buf[0]=='l'){ Serial.print("left");
Serial.println(buf[1]);
digitalWrite(in1, LOW);
digitalWrite(in2, HIGH);
analogWrite(enA,255);
digitalWrite(in3, LOW);
digitalWrite(in4, HIGH);
analogWrite(enB,255);
}
if(buf[0]=='r'){ Serial.print("right");
Serial.println(buf[1]);
digitalWrite(in1, HIGH);
digitalWrite(in2, LOW);
analogWrite(enA,255);
digitalWrite(in3, HIGH);
digitalWrite(in4, LOW);
analogWrite(enB,255);
}
}
}
GESTURE MOTION TRANSMITTER
//connection for DNPU6 //VCC = 3.3/5v //GND = GND //SCL = 3 //SDA = 2 //INT = 7
//connection for transmitter //DATA=12 //VCC=5V //GND=GND
//#include char *controller; #include #include
//#include
MPU6050 mpu;
void setup()
//The Transmitter si sending recieving the code for servo by our Head Set and then transferring it to the reciever
{
pinMode(13,OUTPUT);
// vw_set_ptt_inverted(true);
// // vw_set_tx_pin(12); // vw_setup(4000);// speed of data transfer Kbps
Serial.begin(9600);
Serial.println("Initialize MPU6050");
while(!mpu.begin(MPU6050_SCALE_2000DPS, MPU6050_RANGE_2G))
{ Serial.println("Could not find a valid MPU6050 sensor, check wiring!");
delay(500);
}
}
void loop()
{
// Read normalized values
Vector normAccel = mpu.readNormalizeAccel();
//normalizing the values as in averaging
// Calculate Pitch & Roll
int pitch = -(atan2(normAccel.XAxis, sqrt(normAccel.YAxis*normAccel.YAxis + normAccel.ZAxis*normAccel.ZAxis))*180.0)/M_PI;
int roll = (atan2(normAccel.YAxis, normAccel.ZAxis)*180.0)/M_PI;
Serial.print(" \nPitch = ");
Serial.print(pitch);
Serial.print(" Roll = ");
Serial.print(roll);
if( pitch > 20){ controller="f" ;
vw_send((uint8_t *)controller, strlen(controller));
vw_wait_tx();
// Wait until the whole message is gone
digitalWrite(13, HIGH);
delay(200);
}
if( pitch < -20){ controller="b" ;
vw_send((uint8_t *)controller, strlen(controller));
vw_wait_tx();
// Wait until the whole message is gone
digitalWrite(13, LOW);
delay(200);
}
if(roll > 20){ controller="l" ;
vw_send((uint8_t *)controller, strlen(controller));
vw_wait_tx();
// Wait until the whole message is gone
digitalWrite(13, HIGH);
delay(200);
}
if(roll < -20){ controller="r" ;
vw_send((uint8_t *)controller, strlen(controller));
vw_wait_tx();
// Wait until the whole message is gone
digitalWrite(13, LOW);
delay(200);
} else { controller="s" ;
vw_send((uint8_t *)controller, strlen(controller));
vw_wait_tx();
// Wait until the whole message is gone
delay(200);
digitalWrite(13, LOW);
}
delay(10);
}
GESTURE MOTION RECIEVER
//connection for rf receiver
//GND = GND
//DATA = 12
//VCC = 5V
#include
//connection for motors
//motor one int enA = 10;
int in1 = 9;
int in2 = 8;
// motor two int enB = 5;
int in3 = 7;
int in4 = 6;
void setup() { vw_set_ptt_inverted(true);
// Required for DR3100 vw_set_rx_pin(12);
vw_setup(4000);
// Bits per sec
//pinMode(13, OUTPUT);
// set all the motor control pins to outputs
pinMode(enA, OUTPUT);
pinMode(enB, OUTPUT);
pinMode(in1, OUTPUT);
pinMode(in2, OUTPUT);
pinMode(in3, OUTPUT);
pinMode(in4, OUTPUT);
Serial.begin(9600); vw_rx_start();
// Start the receiver PLL running
}
void loop()
{
uint8_t buf[VW_MAX_MESSAGE_LEN];
uint8_t buflen = 2;
if (vw_get_message(buf, &buflen))
// Non-blocking
{
Serial.print(buf[0]) ;
Serial.println(buf[1]);
if(buf[0]=='f'){
Serial.print("forward");
Serial.println(buf[1]);
digitalWrite(in1, LOW);
digitalWrite(in2, HIGH);
analogWrite(enA,255);
digitalWrite(in3, HIGH);
digitalWrite(in4, LOW);
analogWrite(enB,255);
}
if(buf[0]=='b') {
Serial.print("backward");
Serial.println(buf[1]);
digitalWrite(in1, HIGH);
digitalWrite(in2, LOW);
analogWrite(enA,255);
digitalWrite(in3, LOW);
digitalWrite(in4, HIGH);
analogWrite(enB,255);
}
if(buf[0]=='s'){ Serial.print("stop");
Serial.println(buf[1]);
digitalWrite(in1, LOW);
digitalWrite(in2, LOW);
analogWrite(enA,255);
digitalWrite(in3, LOW);
digitalWrite(in4, LOW);
analogWrite(enB,255);
}
if(buf[0]=='l'){ Serial.print("left");
Serial.println(buf[1]);
digitalWrite(in1, LOW);
digitalWrite(in2, HIGH);
analogWrite(enA,255);
digitalWrite(in3, LOW);
digitalWrite(in4, HIGH);
analogWrite(enB,255);
} if(buf[0]=='r'){ Serial.print("right");
Serial.println(buf[1]);
digitalWrite(in1, HIGH);
digitalWrite(in2, LOW);
analogWrite(enA,255);
digitalWrite(in3, HIGH);
digitalWrite(in4, LOW);
analogWrite(enB,255);
}
}
}
Step 5: Procedure
STEP 1: ASSEMBLING THE BASE
A wooden base was taken in which tires were fitted. The wooden base was designed as a car base. Some slots were made for keeping the Arduino and Power Supply in place. A wooden Rod was fitted vertically over the centre of the base that would support the Camera and Servo Motors. A portion of the top of the Wooden Stick was hollowed out to support the Servos.
STEP 2: MOTION
Two Motors were used for the rear wheel tires. Motor Driver(L298N) was connected to the motors for control in speed and turns. Arduino was connected to the motor driver to control it.
STEP 3: GESTURE CONTROL
We used a Glove in which MPU6050 Sensor and RF Transmitter was attached. We used MPU6050 as Accelerometer and Gyrometer and was attached to the RF Transmitter which was transferring the data based on our hand motions. The RF transmitter was connected to an Arduino Mini. The orientation of the hand decides the pitch and the roll. The Gestures we used were:
* Left – Left Movement
* Right – Right Movement
* Up – Backward Movement
* Down – Forward Movement
* Stable – No Movement
There was an RF reciever which was connected to an Arduino Mini attached to the bot. It was recieving the data for specific movement.
STEP 4: HEAD MOVEMENT (GESTURE CONTROLLED)
We used a Occulus in which MPU6050 Sensor and NRF Transmitter was attached. We used MPU6050 as Accelerometer and Gyrometer and was attached to the NRF Transmitter which was transferring the data based on our head motions. The NRF transmitter was connected to an Arduino UNO. The orientation of the head decides the pitch and the roll. The Gestures we used were:
* Left – Left Movement
* Right – Right Movement
* Up – Upward Movement
* Down – Downward Movement
* Stable – No Movement
NRF Reciever was recieving the data motions of our heads. The reciever was connected to the Servo motor which was present on the top part of the wooden stick. The WebCam was attached to the servo motor for respective motions. Servo and Reciever were connected to Arduino UNO and Webcam was connected to Raspberry Pi with the respective connections.
STEP 5: Video Transfer
The video transfer was from the Raspberry-Pi to the phone we were using. Further we created a local host server so that anyone who has the IP address of that server can see the video. The Phone was placed in the Oculus as well for VR VISION.
STEP 6: Power Supply
When all the connections were done and were uptight, then four 3V batteries were connected to the two respective Arduinos and One Li-PO was used to run the motor driver.
Participated in the
Microcontroller Contest
