// Code for p5.js
// Drawboard Reference: <http://makeabilitylab.github.io/physcomp/communication/p5js-paint-io> By Jon E. Froehlich
let pHtmlMsg;
let serialOptions = { baudRate: 115200 };
let serial;
let mapBrushTypeToShapeName = {
0: "Circle",
1: "Square",
2: "Triangle"
};
let mapBrushFillMode = {
0: "Fill",
1: "Outline",
};
const MAX_BRUSH_SIZE = 150; // the maximum brush size
let brushType = 0; // Circle as default
let brushFillMode = 0; // Fill as default
let brushSize = 20; // Initial brush size
let brushX = 0; // Current brush x location (in pixel coordinates)
let brushY = 0; // Current brush y location (in pixel coordinates)
let brushColor = 0; // Current brush color
let lastBrushX = 0; // Last brush y position (similar to pmouseX but for the brush)
let lastBrushY = 0; // Last brush y position (similar to pmouseY but for the brush)
let showInstructions = true; // If true, shows the app instructions on the screen
// We will paint to an offscreen graphics buffer
// See: <https://p5js.org/reference/#/p5/createGraphics>
let offscreenGfxBuffer;
let p5Canvas;
let isMouseOverCanvas = false;
let hasMouseEnteredCanvas = false;
let myFaceModel;
let myVideo;
let myResults;
const options ={
withLandmakrs: true,
withDesctiptors: false
}
var mouthDistance;
//_____________________________________________
function setup() {
p5Canvas = createCanvas(640, 480);
myVideo = createCapture(VIDEO);
myVideo.hide();
myFaceModel = ml5.faceApi(myVideo, options, modelLoaded);
brushColor = color(250, 250, 250, 80);
// Capture when mouse enters canvas to avoid drawing extraneous
// shapes (especially happens on initialization when mouseX, mouseY
// are both zero)
p5Canvas.mouseOver(onCanvasMouseOver);
p5Canvas.mouseOut(onCanvasMouseOut);
// Setup Web Serial using serial.js
serial = new Serial();
serial.on(SerialEvents.CONNECTION_OPENED, onSerialConnectionOpened);
serial.on(SerialEvents.CONNECTION_CLOSED, onSerialConnectionClosed);
serial.on(SerialEvents.DATA_RECEIVED, onSerialDataReceived);
serial.on(SerialEvents.ERROR_OCCURRED, onSerialErrorOccurred);
// If we have previously approved ports, attempt to connect with them
// serial.autoConnectAndOpenPreviouslylApprovedPort(serialOptions);
// Add in a lil <p> element to provide messages. This is optional
pHtmlMsg = createP("Draw by using the mouse!");
// Initialize the brush color
// Rather than storing individual paint strokes + paint properties in a
// data structure, we simply draw immediately to an offscreen buffer
// and then show this offscreen buffer on each draw call
// See: <https://p5js.org/reference/#/p5/createGraphics>
offscreenGfxBuffer = createGraphics(width, height);
offscreenGfxBuffer.background(60);
}
function modelLoaded(){
console.log ('myFaceModel', myFaceModel);
myFaceModel.detect(gotResults);
}
function gotResults(error, results){
if (error){
console.log('error', error)
}
if (results){
myResults = results;
myFaceModel.detect(gotResults);
}
}
function draw() {
// image(myVideo, 0, 0, width, height);
if (myResults && myResults[0]){
const allMouth = myResults[0].parts.mouth;
for (let i=0; i<allMouth.length; i++){
textSize (14);
mouthDistance = Math.abs (allMouth[14]._y-allMouth[18]._y);
// console.log(mouthDistance);
}
}
brushSize = 10 + floor(mouthDistance * 2);
if (mouthDistance > 10){
brushColor = color(220, 133, 255, 80);
}
else {
brushColor = color(250, 250, 250, 80);
}
// Draw the current brush stroke at the given mouse x, y position
drawBrushStroke(mouseX, mouseY);
// brushColor = color(r, g, b, a);
// console.log(r, g, b, a);
if(serial.isOpen()){
// Draw current brush stroke at current brush x,y position (from serial)
drawBrushStroke(brushX, brushY);
}
// Draw the offscreen buffer to the screen
image(offscreenGfxBuffer, 0, 0);
// Check to see if we are supposed to draw our instructions
if(showInstructions){
drawInstructions();
}
}
/**
* Draws the brush stroke with the current global settings at the x,y position
*
* @param {Number} xBrush x brush position in pixels
* @param {Number} yBrush y brush position in pixels
*/
function drawBrushStroke(xBrush, yBrush){
if(brushSize > 0 && hasMouseEnteredCanvas){
if (brushFillMode == 0) { // brushFillMode 0 is fill
offscreenGfxBuffer.fill(brushColor);
offscreenGfxBuffer.noStroke();
} else { // brushFillMode 0 is outline
offscreenGfxBuffer.stroke(brushColor);
offscreenGfxBuffer.noFill();
}
let xCenter = xBrush;
let yCenter = yBrush;
let halfShapeSize = brushSize / 2;
switch (brushType) {
case 0:
offscreenGfxBuffer.circle(xCenter, yCenter, brushSize);
break;
case 1:
// Draw rectangle based on center coordinates
// <https://p5js.org/reference/#/p5/rectMode>
offscreenGfxBuffer.rectMode(CENTER);
offscreenGfxBuffer.square(xCenter, yCenter, brushSize);
break;
case 2:
let x1 = xCenter - halfShapeSize;
let y1 = yCenter + halfShapeSize;
let x2 = xCenter;
let y2 = yCenter - halfShapeSize;
let x3 = xCenter + halfShapeSize;
let y3 = y1;
offscreenGfxBuffer.triangle(x1, y1, x2, y2, x3, y3)
}
}
}
/**
* Draws the keyboard instructions to the screen
*/
function drawInstructions(){
// Some instructions to the user
noStroke();
fill(255);
let tSize = 10;
textSize(tSize);
let yText = 2;
let yBuffer = 1;
let xText = 3;
text("KEYBOARD COMMANDS", xText, yText + tSize);
yText += tSize + yBuffer;
text("'i' : Show/hide instructions", xText, yText + tSize);
yText += tSize + yBuffer;
text("'l' : Clear the screen", xText, yText + tSize);
yText += tSize + yBuffer;
let strConnectToSerial = "'o' : Open serial (";
if(serial.isOpen()){
strConnectToSerial += "connected";
}else{
strConnectToSerial += "not connected";
}
strConnectToSerial += ")";
text(strConnectToSerial, xText, yText + tSize);
yText += tSize + yBuffer;
let strBrushType = "'b' : Set brush type (" + mapBrushTypeToShapeName[brushType].toLowerCase() + ")";
text(strBrushType, xText, yText + tSize);
yText += tSize + yBuffer;
let strToggleFillMode = "'f' : Toggle fill mode (" + mapBrushFillMode[brushFillMode].toLowerCase() + ")";
text(strToggleFillMode, xText, yText + tSize);
}
/**
* The keyPressed() function is called once every time a key is pressed.
* See: <https://p5js.org/reference/#/p5/keyPressed>
*/
function keyPressed() {
let lastFillMode = brushFillMode;
let lastBrushType = brushType;
print("keyPressed", key);
if(key == 'f'){
brushFillMode++;
if (brushFillMode >= Object.keys(mapBrushFillMode).length) {
brushFillMode = 0;
}
}else if(key == 'b'){
brushType++;
if (brushType >= Object.keys(mapBrushTypeToShapeName).length) {
brushType = 0;
}
}else if(key == 'i'){
showInstructions = !showInstructions;
}else if(key == 'l'){
// To clear the screen, simply "draw" over the existing
// graphics buffer with an empty background
offscreenGfxBuffer.background(60);
}else if(key == 'o'){
if (!serial.isOpen()) {
serial.connectAndOpen(null, serialOptions);
}
}
}
/**
* Callback function by serial.js when there is an error on web serial
*
* @param {} eventSender
*/
function onSerialErrorOccurred(eventSender, error) {
console.log("onSerialErrorOccurred", error);
pHtmlMsg.html(error);
}
/**
* Callback function by serial.js when web serial connection is opened
*
* @param {} eventSender
*/
function onSerialConnectionOpened(eventSender) {
console.log("onSerialConnectionOpened");
pHtmlMsg.html("Serial connection opened successfully");
}
/**
* Callback function by serial.js when web serial connection is closed
*
* @param {} eventSender
*/
function onSerialConnectionClosed(eventSender) {
console.log("onSerialConnectionClosed");
pHtmlMsg.html("onSerialConnectionClosed");
}
/**
* Callback function serial.js when new web serial data is received
*
* @param {*} eventSender
* @param {String} newData new data received over serial
*/
function onSerialDataReceived(eventSender, newData) {
//console.log("onSerialDataReceived", newData);
pHtmlMsg.html("onSerialDataReceived: " + newData);
if(!newData.startsWith("#")){
let startIndex = 0;
let endIndex = newData.indexOf(",");
if(endIndex != -1){
// Parse x location (normalized between 0 and 1)
let strBrushXFraction = newData.substring(startIndex, endIndex).trim();
let xFraction = parseFloat(strBrushXFraction);
// Parse y location (normalized between 0 and 1)
startIndex = endIndex + 1;
endIndex = newData.length;
let strBrushYFraction = newData.substring(startIndex, endIndex).trim();
let yFraction = parseFloat(strBrushYFraction);
// Set relevant global variables for brush x,y location in pixels
lastBrushX = brushX;
lastBrushY = brushY;
brushX = xFraction * width;
brushY = yFraction * height;
}
}
}
function onCanvasMouseOver(){
hasMouseEnteredCanvas = true;
isMouseOverCanvas = true;
}
function onCanvasMouseOut(){
isMouseOverCanvas = false;
}
// Index file
<!DOCTYPE html>
<html lang="en">
<head>
<script src="<https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.7.0/p5.js>"></script>
<script src="<https://unpkg.com/ml5@latest/dist/ml5.min.js>"></script>
<script src="<https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.7.0/addons/p5.sound.min.js>"></script>
<script src="<https://cdn.jsdelivr.net/gh/makeabilitylab/p5js/_libraries/serial.js>"></script>
<link rel="stylesheet" type="text/css" href="css\style.css">
<meta charset="utf-8" />
</head>
<body>
<script src="sketch.js"></script>
</body>
</html>
//Code for arduino
const int X_ANALOG_INPUT_PIN = A0;
const int Y_ANALOG_INPUT_PIN = A1;
// On the Arduino Uno/Leonardo, there is a 10-bit ADC so the
// maximum analog value is 1023. On other microcontrollers, like the ESP32,
// there is a 12-bit ADC, so the maximum analog value is 4095
const int MAX_ANALOG_VAL = 1023;
const long BAUD_RATE = 115200;
void setup() {
Serial.begin(BAUD_RATE);
}
void loop() {
// Read the analog values
int xAnalogVal = analogRead(X_ANALOG_INPUT_PIN);
int yAnalogVal = analogRead(Y_ANALOG_INPUT_PIN);
// Calculate normalized x,y location
float xNormalized = xAnalogVal / (float)MAX_ANALOG_VAL;
float yNormalized = yAnalogVal / (float)MAX_ANALOG_VAL;
// Transmit over serial as comma-separated string
Serial.print(xNormalized, 4);
Serial.print(", ");
Serial.println(yNormalized, 4);
delay(10);
}