Skip to content

Commit 1648438

Browse files
AlmasBAlmasB
committed
feat: added face detection API
1 parent f08351d commit 1648438

5 files changed

Lines changed: 248 additions & 0 deletions

File tree

‎fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/WebAPI.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,13 @@ public final class WebAPI {
3333
public static final URL SPEECH_RECOGNITION_API = URLS.get("speechrecog/index.html");
3434
public static final URL GESTURE_RECOGNITION_API = URLS.get("gesturerecog/index.html");
3535
public static final URL HAND_TRACKING_API = URLS.get("handtracking/index.html");
36+
public static final URL FACE_DETECTION_API = URLS.get("facedetect/index.html");
3637

3738
public static final int TEXT_TO_SPEECH_PORT = 55550;
3839
public static final int SPEECH_RECOGNITION_PORT = 55555;
3940
public static final int GESTURE_RECOGNITION_PORT = 55560;
4041
public static final int HAND_TRACKING_PORT = 55565;
42+
public static final int FACE_DETECTION_PORT = 55570;
4143

4244
private static Map<String, URL> extractURLs() {
4345
var map = new HashMap<String, URL>();
@@ -50,6 +52,8 @@ private static Map<String, URL> extractURLs() {
5052
"gesturerecog/script.js",
5153
"handtracking/index.html",
5254
"handtracking/script.js",
55+
"facedetect/index.html",
56+
"facedetect/script.js",
5357
"speechrecog/index.html",
5458
"speechrecog/script.js"
5559
).forEach(relativeURL -> {

‎fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/facedetect/Face.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* FXGL - JavaFX Game Library. The MIT License (MIT).
3+
* Copyright (c) AlmasB (almaslvl@gmail.com).
4+
* See LICENSE for details.
5+
*/
6+
7+
package com.almasb.fxgl.intelligence.facedetect
8+
9+
/**
10+
* @author Almas Baim (https://github.com/AlmasB)
11+
*/
12+
data class Face(
13+
val id: Int,
14+
val x: Int,
15+
val y: Int,
16+
val width: Int,
17+
val height: Int
18+
)

‎fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/facedetect/FaceDetectionFromImageService.kt‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* FXGL - JavaFX Game Library. The MIT License (MIT).
3+
* Copyright (c) AlmasB (almaslvl@gmail.com).
4+
* See LICENSE for details.
5+
*/
6+
7+
package com.almasb.fxgl.intelligence.facedetect
8+
9+
import com.almasb.fxgl.core.concurrent.Async
10+
import com.almasb.fxgl.intelligence.WebAPI
11+
import com.almasb.fxgl.intelligence.WebAPIService
12+
import com.almasb.fxgl.logging.Logger
13+
import com.almasb.fxgl.net.ws.LocalWebSocketServer
14+
import com.almasb.fxgl.texture.toBase64
15+
import javafx.scene.image.Image
16+
import java.util.function.Consumer
17+
18+
/**
19+
* Service that provides access to face detection from a given image.
20+
*
21+
* @author Almas Baim (https://github.com/AlmasB)
22+
*/
23+
class FaceDetectionFromImageService : WebAPIService(
24+
LocalWebSocketServer("FaceDetectionServer", WebAPI.FACE_DETECTION_PORT),
25+
WebAPI.FACE_DETECTION_API
26+
) {
27+
28+
private val log = Logger.get(FaceDetectionFromImageService::class.java)
29+
30+
private val faceDataHandlers = arrayListOf<Consumer<Face>>()
31+
32+
private fun initService() {
33+
log.debug("initService()")
34+
35+
setReady()
36+
}
37+
38+
private fun onFaceInput(message: String) {
39+
try {
40+
val rawData = message.split(",").filter { it.isNotEmpty() }
41+
42+
val id = rawData[0].toInt()
43+
val x = rawData[1].toInt()
44+
val y = rawData[2].toInt()
45+
val w = rawData[3].toInt()
46+
val h = rawData[4].toInt()
47+
val score = rawData[5].toDouble()
48+
49+
Async.startAsyncFX {
50+
faceDataHandlers.forEach { it.accept(Face(id, x, y, w, h)) }
51+
}
52+
53+
} catch (e: Exception) {
54+
log.warning("Failed to parse message.", e)
55+
}
56+
}
57+
58+
/**
59+
* Add input handler for face recognition data.
60+
* Input handlers are called on the JavaFX thread.
61+
*/
62+
fun addInputHandler(handler: Consumer<Face>) {
63+
faceDataHandlers += handler
64+
}
65+
66+
fun removeInputHandler(handler: Consumer<Face>) {
67+
faceDataHandlers -= handler
68+
}
69+
70+
fun detect(image: Image) {
71+
Async.startAsync {
72+
rpcRun("detect", "data:image/png;base64," + image.toBase64())
73+
}
74+
}
75+
}

‎fxgl-intelligence/src/main/resources/com/almasb/fxgl/intelligence/facedetect/index.html‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<!-- Copyright 2023 The MediaPipe Authors.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. -->
14+
15+
<!DOCTYPE html>
16+
<html>
17+
<head>
18+
<meta charset="utf-8">
19+
<script src="https://unpkg.com/material-components-web@latest/dist/material-components-web.min.js"></script>
20+
<script type="module" src="https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/vision_bundle.js" crossorigin="anonymous"></script>
21+
<script type="module" src="script.js"></script>
22+
</head>
23+
24+
<body>
25+
<h2>Hand tracking service</h2>
26+
27+
<section id="demos" class="invisible" hidden>
28+
<div>
29+
<img id="inputImage" crossorigin="anonymous">
30+
</div>
31+
</section>
32+
33+
</body>
34+
</html>
35+
36+

‎fxgl-intelligence/src/main/resources/com/almasb/fxgl/intelligence/facedetect/script.js‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Copyright 2023 The MediaPipe Authors.
2+
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import {
16+
FaceDetector,
17+
FilesetResolver
18+
} from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0";
19+
20+
let faceDetector = undefined;
21+
let isReady = false;
22+
23+
let inputImage = document.getElementById("inputImage")
24+
25+
const socket = new WebSocket('ws://localhost:55570');
26+
27+
socket.addEventListener('open', function (event) {
28+
createFaceDetector();
29+
});
30+
31+
socket.addEventListener('message', function (event) {
32+
let message = event.data;
33+
34+
if (message.startsWith(FUNCTION_CALL_TAG)) {
35+
let func = message.substring(FUNCTION_CALL_TAG.length);
36+
let tokens = func.split('*,,*');
37+
let funcName = tokens[0];
38+
39+
if (funcName === "detect") {
40+
let image64 = tokens[1];
41+
42+
detect(image64);
43+
}
44+
}
45+
});
46+
47+
const createFaceDetector = async () => {
48+
const vision = await FilesetResolver.forVisionTasks(
49+
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"
50+
);
51+
52+
faceDetector = await FaceDetector.createFromOptions(vision, {
53+
baseOptions: {
54+
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite`,
55+
delegate: "GPU"
56+
},
57+
runningMode: "IMAGE"
58+
});
59+
60+
inputImage.addEventListener("load", handleClick);
61+
rpcRun("initService");
62+
63+
isReady = true;
64+
};
65+
66+
function detect(image64) {
67+
inputImage.setAttribute("src", image64);
68+
}
69+
70+
async function handleClick(event) {
71+
const detections = faceDetector.detect(inputImage).detections;
72+
73+
console.log(detections);
74+
75+
if (detections) {
76+
var id = 0;
77+
for (const detection of detections) {
78+
var data = "" + id + ",";
79+
80+
data += detection.boundingBox.originX + ",";
81+
data += detection.boundingBox.originY + ",";
82+
data += detection.boundingBox.width + ",";
83+
data += detection.boundingBox.height + ",";
84+
data += parseFloat(detection.categories[0].score);
85+
86+
rpcRun("onFaceInput", data);
87+
88+
id++;
89+
}
90+
}
91+
}
92+
93+
// the below is a copy-paste since this js file is a module but ../rpc-common.js is not
94+
// include ../rpc-common.js
95+
96+
const SEPARATOR = "*,,*";
97+
const FUNCTION_CALL_TAG = "F_CALL:";
98+
const FUNCTION_RETURN_TAG = "F_RETURN:";
99+
100+
function rpcRun(funcName, ...args) {
101+
let argsString = "";
102+
103+
for (const arg of args) {
104+
argsString += arg + SEPARATOR;
105+
}
106+
107+
let message = `${FUNCTION_CALL_TAG}${funcName}${SEPARATOR}${argsString}`;
108+
109+
socket.send(message);
110+
}
111+
112+
function rpcReturn(funcName) {
113+
// TODO: unique id?
114+
//socket.send(`${FUNCTION_RETURN_TAG}${funcName}.F_RESULT:${names}`);
115+
}

0 commit comments

Comments
 (0)