mirror of
https://github.com/huggingface/lerobot.git
synced 2026-06-04 04:41:24 +00:00
Compare commits
4 Commits
accelerate
...
mrussi/glo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4c17a7be2b | ||
|
|
c8c37bd339 | ||
|
|
0ccc08e347 | ||
|
|
7b207d44a0 |
@@ -1,11 +0,0 @@
|
|||||||
compute_environment: LOCAL_MACHINE
|
|
||||||
debug: false
|
|
||||||
distributed_type: NO
|
|
||||||
downcast_bf16: 'no'
|
|
||||||
enable_cpu_affinity: false
|
|
||||||
machine_rank: 0
|
|
||||||
main_training_function: main
|
|
||||||
mixed_precision: 'no'
|
|
||||||
num_machines: 1
|
|
||||||
num_processes: 1
|
|
||||||
use_cpu: false
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
compute_environment: LOCAL_MACHINE
|
|
||||||
debug: false
|
|
||||||
distributed_type: MULTI_GPU
|
|
||||||
downcast_bf16: 'no'
|
|
||||||
enable_cpu_affinity: false
|
|
||||||
gpu_ids: all
|
|
||||||
machine_rank: 0
|
|
||||||
main_training_function: main
|
|
||||||
mixed_precision: 'no'
|
|
||||||
num_machines: 1
|
|
||||||
num_processes: 2
|
|
||||||
rdzv_backend: static
|
|
||||||
same_network: true
|
|
||||||
tpu_env: []
|
|
||||||
tpu_use_cluster: false
|
|
||||||
tpu_use_sudo: false
|
|
||||||
use_cpu: false
|
|
||||||
dynamo_backend: "no"
|
|
||||||
@@ -29,7 +29,7 @@ ENV DEBIAN_FRONTEND=noninteractive \
|
|||||||
|
|
||||||
# Install system dependencies and uv (as root)
|
# Install system dependencies and uv (as root)
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
build-essential git curl libglib2.0-0 libegl1-mesa-dev ffmpeg \
|
build-essential git curl libglib2.0-0 libegl1-mesa ffmpeg \
|
||||||
libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \
|
libusb-1.0-0-dev speech-dispatcher libgeos-dev portaudio19-dev \
|
||||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
|
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
|
||||||
&& mv /root/.local/bin/uv /usr/local/bin/uv \
|
&& mv /root/.local/bin/uv /usr/local/bin/uv \
|
||||||
|
|||||||
@@ -39,8 +39,6 @@
|
|||||||
- sections:
|
- sections:
|
||||||
- local: notebooks
|
- local: notebooks
|
||||||
title: Notebooks
|
title: Notebooks
|
||||||
- local: feetech
|
|
||||||
title: Updating Feetech Firmware
|
|
||||||
title: "Resources"
|
title: "Resources"
|
||||||
- sections:
|
- sections:
|
||||||
- local: contributing
|
- local: contributing
|
||||||
|
|||||||
@@ -1,71 +0,0 @@
|
|||||||
# Feetech Motor Firmware Update
|
|
||||||
|
|
||||||
This tutorial guides you through updating the firmware of Feetech motors using the official Feetech software.
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- Windows computer (Feetech software is only available for Windows)
|
|
||||||
- Feetech motor control board
|
|
||||||
- USB cable to connect the control board to your computer
|
|
||||||
- Feetech motors connected to the control board
|
|
||||||
|
|
||||||
## Step 1: Download Feetech Software
|
|
||||||
|
|
||||||
1. Visit the official Feetech software download page: [https://www.feetechrc.com/software.html](https://www.feetechrc.com/software.html)
|
|
||||||
2. Download the latest version of the Feetech debugging software (FD)
|
|
||||||
3. Install the software on your Windows computer
|
|
||||||
|
|
||||||
## Step 2: Hardware Setup
|
|
||||||
|
|
||||||
1. Connect your Feetech motors to the motor control board
|
|
||||||
2. Connect the motor control board to your Windows computer via USB cable
|
|
||||||
3. Ensure power is supplied to the motors
|
|
||||||
|
|
||||||
## Step 3: Configure Connection
|
|
||||||
|
|
||||||
1. Launch the Feetech debugging software
|
|
||||||
2. Select the correct COM port from the port dropdown menu
|
|
||||||
- If unsure which port to use, check Windows Device Manager under "Ports (COM & LPT)"
|
|
||||||
3. Set the appropriate baud rate (typically 1000000 for most Feetech motors)
|
|
||||||
4. Click "Open" to establish communication with the control board
|
|
||||||
|
|
||||||
## Step 4: Scan for Motors
|
|
||||||
|
|
||||||
1. Once connected, click the "Search" button to detect all connected motors
|
|
||||||
2. The software will automatically discover and list all motors on the bus
|
|
||||||
3. Each motor will appear with its ID number
|
|
||||||
|
|
||||||
## Step 5: Update Firmware
|
|
||||||
|
|
||||||
For each motor you want to update:
|
|
||||||
|
|
||||||
1. **Select the motor** from the list by clicking on it
|
|
||||||
2. **Click on Upgrade tab**:
|
|
||||||
3. **Click on Online button**:
|
|
||||||
- If an potential firmware update is found, it will be displayed in the box
|
|
||||||
4. **Click on Upgrade button**:
|
|
||||||
- The update progress will be displayed
|
|
||||||
|
|
||||||
## Step 6: Verify Update
|
|
||||||
|
|
||||||
1. After the update completes, the software should automatically refresh the motor information
|
|
||||||
2. Verify that the firmware version has been updated to the expected version
|
|
||||||
|
|
||||||
## Important Notes
|
|
||||||
|
|
||||||
⚠️ **Warning**: Do not disconnect power or USB during firmware updates, it will potentially brick the motor.
|
|
||||||
|
|
||||||
## Bonus: Motor Debugging on Linux/macOS
|
|
||||||
|
|
||||||
For debugging purposes only, you can use the open-source Feetech Debug Tool:
|
|
||||||
|
|
||||||
- **Repository**: [FT_SCServo_Debug_Qt](https://github.com/CarolinePascal/FT_SCServo_Debug_Qt/tree/fix/port-search-timer)
|
|
||||||
|
|
||||||
### Installation Instructions
|
|
||||||
|
|
||||||
Follow the instructions in the repository to install the tool, for Ubuntu you can directly install it, for MacOS you need to build it from source.
|
|
||||||
|
|
||||||
**Limitations:**
|
|
||||||
|
|
||||||
- This tool is for debugging and parameter adjustment only
|
|
||||||
- Firmware updates must still be done on Windows with official Feetech software
|
|
||||||
@@ -117,6 +117,14 @@ middle_dip | 1484 | 1500 | 1547
|
|||||||
|
|
||||||
Once calibration is complete, the system will save the calibration to `/Users/your_username/.cache/huggingface/lerobot/calibration/teleoperators/homunculus_glove/red.json`
|
Once calibration is complete, the system will save the calibration to `/Users/your_username/.cache/huggingface/lerobot/calibration/teleoperators/homunculus_glove/red.json`
|
||||||
|
|
||||||
|
#### Visualizing Teleoperator Glove
|
||||||
|
|
||||||
|
After calibration, you can visualize the glove movements in real-time. Open the visualizer by navigating to the visualizer directory and opening the HTML file in your browser:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
open examples/hopejr/visualizer/index.html
|
||||||
|
```
|
||||||
|
|
||||||
### 1.3 Calibrate Robot Arm
|
### 1.3 Calibrate Robot Arm
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
182
examples/hopejr/visualizer/index.html
Normal file
182
examples/hopejr/visualizer/index.html
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>3D Hand Joint Visualizer</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-family: Arial, sans-serif;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
height: 100vh;
|
||||||
|
}
|
||||||
|
|
||||||
|
.controls {
|
||||||
|
padding: 15px;
|
||||||
|
background-color: #f5f5f5;
|
||||||
|
z-index: 100;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status {
|
||||||
|
padding: 10px;
|
||||||
|
border-radius: 5px;
|
||||||
|
margin: 10px 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connected {
|
||||||
|
background-color: #d4edda;
|
||||||
|
color: #155724;
|
||||||
|
}
|
||||||
|
|
||||||
|
.disconnected {
|
||||||
|
background-color: #f8d7da;
|
||||||
|
color: #721c24;
|
||||||
|
}
|
||||||
|
|
||||||
|
button {
|
||||||
|
padding: 8px 16px;
|
||||||
|
background-color: #4CAF50;
|
||||||
|
color: white;
|
||||||
|
border: none;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
margin-right: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:hover {
|
||||||
|
background-color: #45a049;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:disabled {
|
||||||
|
background-color: #cccccc;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.container {
|
||||||
|
display: flex;
|
||||||
|
flex: 1;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
#canvas-container {
|
||||||
|
flex: 3;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sidebar {
|
||||||
|
flex: 1;
|
||||||
|
padding: 15px;
|
||||||
|
background-color: #f8f9fa;
|
||||||
|
overflow-y: auto;
|
||||||
|
max-width: 300px;
|
||||||
|
border-left: 1px solid #ddd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.joint-info {
|
||||||
|
margin-bottom: 10px;
|
||||||
|
padding: 8px;
|
||||||
|
border: 1px solid #ddd;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.joint-name {
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.joint-value {
|
||||||
|
font-family: monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bar-container {
|
||||||
|
width: 100%;
|
||||||
|
background-color: #e0e0e0;
|
||||||
|
height: 10px;
|
||||||
|
border-radius: 5px;
|
||||||
|
overflow: hidden;
|
||||||
|
margin-top: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bar {
|
||||||
|
height: 100%;
|
||||||
|
background-color: #4CAF50;
|
||||||
|
width: 0%;
|
||||||
|
transition: width 0.2s ease-in-out;
|
||||||
|
}
|
||||||
|
|
||||||
|
.log-container {
|
||||||
|
margin-top: 20px;
|
||||||
|
border: 1px solid #ddd;
|
||||||
|
border-radius: 5px;
|
||||||
|
padding: 10px;
|
||||||
|
height: 150px;
|
||||||
|
overflow-y: auto;
|
||||||
|
font-family: monospace;
|
||||||
|
background-color: #f8f9fa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.view-controls {
|
||||||
|
position: absolute;
|
||||||
|
bottom: 10px;
|
||||||
|
left: 10px;
|
||||||
|
z-index: 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
.view-button {
|
||||||
|
background-color: rgba(0, 0, 0, 0.5);
|
||||||
|
color: white;
|
||||||
|
border: none;
|
||||||
|
padding: 5px 10px;
|
||||||
|
margin-right: 5px;
|
||||||
|
border-radius: 3px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="controls">
|
||||||
|
<button id="connectButton">Connect to Device</button>
|
||||||
|
<button id="disconnectButton" disabled>Disconnect</button>
|
||||||
|
<select id="baudRate">
|
||||||
|
<option value="9600">9600</option>
|
||||||
|
<option value="19200">19200</option>
|
||||||
|
<option value="38400">38400</option>
|
||||||
|
<option value="57600">57600</option>
|
||||||
|
<option value="115200" selected>115200</option>
|
||||||
|
</select>
|
||||||
|
<span id="statusIndicator" class="status disconnected">Status: Disconnected</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="container">
|
||||||
|
<div id="canvas-container">
|
||||||
|
<!-- 3D canvas will be inserted here -->
|
||||||
|
<div class="view-controls">
|
||||||
|
<button class="view-button" id="frontView">Front</button>
|
||||||
|
<button class="view-button" id="sideView">Side</button>
|
||||||
|
<button class="view-button" id="topView">Top</button>
|
||||||
|
<button class="view-button" id="resetView">Reset</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="sidebar">
|
||||||
|
<h3>Joint Values</h3>
|
||||||
|
<div id="jointsContainer">
|
||||||
|
<!-- Joint info will be added here -->
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="log-container" id="logContainer">
|
||||||
|
<!-- Log messages will be added here -->
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Import Three.js -->
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/three@0.132.2/build/three.min.js"></script>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/three@0.132.2/examples/js/controls/OrbitControls.js"></script>
|
||||||
|
|
||||||
|
<script src="script.js"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
669
examples/hopejr/visualizer/script.js
Normal file
669
examples/hopejr/visualizer/script.js
Normal file
@@ -0,0 +1,669 @@
|
|||||||
|
// === Hand Visualizer with Pre-Connect Sliders + Per-Joint Angle Limits ===
|
||||||
|
// Assumes your HTML already has elements with the following IDs:
|
||||||
|
// connectButton, disconnectButton, baudRate, statusIndicator, jointsContainer, logContainer,
|
||||||
|
// canvas-container, frontView, sideView, topView, resetView
|
||||||
|
// Requires Three.js + OrbitControls loaded on the page.
|
||||||
|
|
||||||
|
// -------------------- Config --------------------
|
||||||
|
const MAX_JOINTS = 16;
|
||||||
|
const RAW_MIN = 0, RAW_MAX = 4096;
|
||||||
|
const RAW_CENTER = (RAW_MIN + RAW_MAX) / 2;
|
||||||
|
const DEG = Math.PI / 180;
|
||||||
|
const UI_DEG_MIN = -90, UI_DEG_MAX = 90; // UI sliders for angle limits
|
||||||
|
|
||||||
|
// -------------------- State --------------------
|
||||||
|
let port;
|
||||||
|
let reader;
|
||||||
|
let keepReading = false;
|
||||||
|
let isConnected = false;
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let inputBuffer = '';
|
||||||
|
|
||||||
|
let jointValues = new Array(MAX_JOINTS).fill(RAW_CENTER);
|
||||||
|
|
||||||
|
// Auto-calibration: track observed min/max per joint
|
||||||
|
let observedMin = new Array(MAX_JOINTS).fill(Infinity);
|
||||||
|
let observedMax = new Array(MAX_JOINTS).fill(-Infinity);
|
||||||
|
let calibrationEnabled = true;
|
||||||
|
|
||||||
|
// Three.js
|
||||||
|
let scene, camera, renderer, controls;
|
||||||
|
let hand = { palm: null, fingers: [] };
|
||||||
|
|
||||||
|
// DOM
|
||||||
|
const connectButton = document.getElementById('connectButton');
|
||||||
|
const disconnectButton = document.getElementById('disconnectButton');
|
||||||
|
const baudRateSelect = document.getElementById('baudRate');
|
||||||
|
const statusIndicator = document.getElementById('statusIndicator');
|
||||||
|
const jointsContainer = document.getElementById('jointsContainer');
|
||||||
|
const logContainer = document.getElementById('logContainer');
|
||||||
|
const canvasContainer = document.getElementById('canvas-container');
|
||||||
|
const frontViewBtn = document.getElementById('frontView');
|
||||||
|
const sideViewBtn = document.getElementById('sideView');
|
||||||
|
const topViewBtn = document.getElementById('topView');
|
||||||
|
const resetViewBtn = document.getElementById('resetView');
|
||||||
|
|
||||||
|
// Helpers
|
||||||
|
const clamp = (x, a, b) => Math.max(a, Math.min(b, x));
|
||||||
|
const invLerp = (a, b, x) => clamp((x - a) / (b - a), 0, 1);
|
||||||
|
|
||||||
|
// -------------------- Joint Map with per-joint angle limits --------------------
|
||||||
|
const fingerJointMap = [
|
||||||
|
// Thumb (4)
|
||||||
|
{ finger:0, joint:0, type:'CMC_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:0, joint:1, type:'CMC_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:0, joint:2, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
{ finger:0, joint:3, type:'IP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
|
||||||
|
// Index (3)
|
||||||
|
{ finger:1, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:1, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:1, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
|
||||||
|
// Middle (3)
|
||||||
|
{ finger:2, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:2, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:2, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
|
||||||
|
// Ring (3)
|
||||||
|
{ finger:3, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:3, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:3, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false }, // +45° only
|
||||||
|
|
||||||
|
// Pinky (3)
|
||||||
|
{ finger:4, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:4, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:4, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false } // +45° only
|
||||||
|
];
|
||||||
|
|
||||||
|
// Assign angle limits (radians) per joint (default ±45°, exceptions: +45° only)
|
||||||
|
for (const j of fingerJointMap) {
|
||||||
|
const isThumb = j.finger === 0;
|
||||||
|
const isPIP = j.type === 'PIP_FLEXION';
|
||||||
|
let minA = -45 * DEG, maxA = +45 * DEG;
|
||||||
|
if ((isThumb && (j.type === 'MCP_FLEXION' || j.type === 'IP_FLEXION')) || (!isThumb && isPIP)) {
|
||||||
|
minA = 0;
|
||||||
|
maxA = +45 * DEG;
|
||||||
|
}
|
||||||
|
j.angleMin = minA;
|
||||||
|
j.angleMax = maxA;
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- UI: Joint Panel --------------------
|
||||||
|
const uiRefs = []; // per joint: { valueLabel, bar, barWrap, slider, invertChk, minDeg, maxDeg }
|
||||||
|
|
||||||
|
function initializeJointElements() {
|
||||||
|
jointsContainer.innerHTML = '';
|
||||||
|
uiRefs.length = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < MAX_JOINTS; i++) {
|
||||||
|
const wrap = document.createElement('div');
|
||||||
|
wrap.className = 'joint-info';
|
||||||
|
|
||||||
|
const fingerIndex = i < 4 ? 0 : Math.floor((i - 4) / 3) + 1;
|
||||||
|
const jointInfo = fingerJointMap[i];
|
||||||
|
const jointType = jointInfo?.type || 'Unknown';
|
||||||
|
const fingerName = ['Thumb', 'Index', 'Middle', 'Ring', 'Pinky'][fingerIndex];
|
||||||
|
|
||||||
|
// Header
|
||||||
|
const nameEl = document.createElement('div');
|
||||||
|
nameEl.className = 'joint-name';
|
||||||
|
nameEl.textContent = `${fingerName} – ${jointType}`;
|
||||||
|
|
||||||
|
// Value + bar
|
||||||
|
const valueEl = document.createElement('div');
|
||||||
|
valueEl.className = 'joint-value';
|
||||||
|
valueEl.textContent = `Value: ${jointValues[i]}`;
|
||||||
|
|
||||||
|
const barWrap = document.createElement('div');
|
||||||
|
barWrap.className = 'bar-container';
|
||||||
|
const barEl = document.createElement('div');
|
||||||
|
barEl.className = 'bar';
|
||||||
|
barWrap.appendChild(barEl);
|
||||||
|
|
||||||
|
// Slider for pre-connect manual control
|
||||||
|
const slider = document.createElement('input');
|
||||||
|
slider.type = 'range';
|
||||||
|
slider.min = String(RAW_MIN);
|
||||||
|
slider.max = String(RAW_MAX);
|
||||||
|
slider.value = String(jointValues[i]);
|
||||||
|
slider.step = '1';
|
||||||
|
slider.className = 'joint-slider';
|
||||||
|
|
||||||
|
slider.addEventListener('input', () => {
|
||||||
|
if (isConnected) return; // ignore while connected
|
||||||
|
let v = parseInt(slider.value, 10);
|
||||||
|
if (jointInfo?.inverted) v = (jointInfo.min + jointInfo.max) - v;
|
||||||
|
jointValues[i] = clamp(jointInfo ? v : 0, RAW_MIN, RAW_MAX);
|
||||||
|
updateJointDisplay(i, jointValues[i]);
|
||||||
|
updateHandModel();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Invert checkbox
|
||||||
|
const invertLbl = document.createElement('label');
|
||||||
|
invertLbl.className = 'invert-toggle';
|
||||||
|
const invertChk = document.createElement('input');
|
||||||
|
invertChk.type = 'checkbox';
|
||||||
|
invertChk.checked = !!jointInfo?.inverted;
|
||||||
|
invertChk.addEventListener('change', () => {
|
||||||
|
if (jointInfo) jointInfo.inverted = invertChk.checked;
|
||||||
|
addLogMessage(`${fingerName} ${jointType} inversion ${invertChk.checked ? 'enabled' : 'disabled'}`);
|
||||||
|
});
|
||||||
|
invertLbl.appendChild(invertChk);
|
||||||
|
invertLbl.appendChild(document.createTextNode('Invert Values'));
|
||||||
|
|
||||||
|
// Angle limits (deg) controls
|
||||||
|
const limitsRow = document.createElement('div');
|
||||||
|
limitsRow.className = 'limits-row';
|
||||||
|
|
||||||
|
const minDeg = document.createElement('input');
|
||||||
|
minDeg.type = 'number';
|
||||||
|
minDeg.min = String(UI_DEG_MIN);
|
||||||
|
minDeg.max = String(UI_DEG_MAX);
|
||||||
|
minDeg.step = '1';
|
||||||
|
minDeg.value = String(Math.round((jointInfo.angleMin || 0) / DEG));
|
||||||
|
minDeg.className = 'limit-num';
|
||||||
|
|
||||||
|
const maxDeg = document.createElement('input');
|
||||||
|
maxDeg.type = 'number';
|
||||||
|
maxDeg.min = String(UI_DEG_MIN);
|
||||||
|
maxDeg.max = String(UI_DEG_MAX);
|
||||||
|
maxDeg.step = '1';
|
||||||
|
maxDeg.value = String(Math.round((jointInfo.angleMax || 0) / DEG));
|
||||||
|
maxDeg.className = 'limit-num';
|
||||||
|
|
||||||
|
const minLbl = document.createElement('span'); minLbl.textContent = 'min°';
|
||||||
|
const maxLbl = document.createElement('span'); maxLbl.textContent = 'max°';
|
||||||
|
minLbl.className = 'limit-label'; maxLbl.className = 'limit-label';
|
||||||
|
|
||||||
|
function syncLimits() {
|
||||||
|
let mn = parseFloat(minDeg.value);
|
||||||
|
let mx = parseFloat(maxDeg.value);
|
||||||
|
if (isNaN(mn)) mn = -45;
|
||||||
|
if (isNaN(mx)) mx = +45;
|
||||||
|
if (mn > mx) [mn, mx] = [mx, mn];
|
||||||
|
jointInfo.angleMin = clamp(mn, UI_DEG_MIN, UI_DEG_MAX) * DEG;
|
||||||
|
jointInfo.angleMax = clamp(mx, UI_DEG_MIN, UI_DEG_MAX) * DEG;
|
||||||
|
minDeg.value = String(Math.round(jointInfo.angleMin / DEG));
|
||||||
|
maxDeg.value = String(Math.round(jointInfo.angleMax / DEG));
|
||||||
|
updateHandModel();
|
||||||
|
}
|
||||||
|
minDeg.addEventListener('change', syncLimits);
|
||||||
|
maxDeg.addEventListener('change', syncLimits);
|
||||||
|
|
||||||
|
limitsRow.appendChild(minLbl);
|
||||||
|
limitsRow.appendChild(minDeg);
|
||||||
|
limitsRow.appendChild(maxLbl);
|
||||||
|
limitsRow.appendChild(maxDeg);
|
||||||
|
|
||||||
|
// Calibration controls
|
||||||
|
const calibRow = document.createElement('div');
|
||||||
|
calibRow.className = 'calib-row';
|
||||||
|
|
||||||
|
const resetCalibBtn = document.createElement('button');
|
||||||
|
resetCalibBtn.textContent = 'Reset Calib';
|
||||||
|
resetCalibBtn.className = 'calib-btn';
|
||||||
|
resetCalibBtn.addEventListener('click', () => {
|
||||||
|
observedMin[i] = Infinity;
|
||||||
|
observedMax[i] = -Infinity;
|
||||||
|
addLogMessage(`Reset calibration for ${fingerName} ${jointType}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
const calibStatus = document.createElement('span');
|
||||||
|
calibStatus.className = 'calib-status';
|
||||||
|
calibStatus.textContent = `Range: --`;
|
||||||
|
|
||||||
|
calibRow.appendChild(resetCalibBtn);
|
||||||
|
calibRow.appendChild(calibStatus);
|
||||||
|
|
||||||
|
// Compose
|
||||||
|
wrap.appendChild(nameEl);
|
||||||
|
wrap.appendChild(valueEl);
|
||||||
|
wrap.appendChild(barWrap);
|
||||||
|
wrap.appendChild(slider);
|
||||||
|
wrap.appendChild(invertLbl);
|
||||||
|
wrap.appendChild(limitsRow);
|
||||||
|
wrap.appendChild(calibRow);
|
||||||
|
|
||||||
|
jointsContainer.appendChild(wrap);
|
||||||
|
|
||||||
|
uiRefs[i] = { valueLabel: valueEl, bar: barEl, barWrap, slider, invertChk, minDeg, maxDeg, nameEl, calibStatus };
|
||||||
|
}
|
||||||
|
|
||||||
|
setConnectedUI(false); // initial state: sliders active
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toggle UI between pre-connect SLIDERS vs post-connect BARS
|
||||||
|
function setConnectedUI(connected) {
|
||||||
|
isConnected = connected;
|
||||||
|
for (let i = 0; i < uiRefs.length; i++) {
|
||||||
|
const ui = uiRefs[i];
|
||||||
|
if (!ui) continue;
|
||||||
|
// Show bars when connected; sliders disabled/hidden
|
||||||
|
ui.barWrap.style.display = connected ? '' : 'none';
|
||||||
|
ui.slider.disabled = connected;
|
||||||
|
ui.slider.style.display = connected ? 'none' : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset calibration when connecting
|
||||||
|
if (connected) {
|
||||||
|
observedMin.fill(Infinity);
|
||||||
|
observedMax.fill(-Infinity);
|
||||||
|
addLogMessage('Calibration reset - move joints through full range for best results');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update joint display (value text + bar color/width + slider position if needed)
|
||||||
|
function updateJointDisplay(jointIndex, value) {
|
||||||
|
const ui = uiRefs[jointIndex];
|
||||||
|
const info = fingerJointMap[jointIndex];
|
||||||
|
if (!ui || !info) return;
|
||||||
|
|
||||||
|
ui.valueLabel.textContent = `Value: ${value}`;
|
||||||
|
|
||||||
|
// bar
|
||||||
|
const min = info.min, max = info.max;
|
||||||
|
const pct = clamp((value - min) / (max - min), 0, 1) * 100;
|
||||||
|
ui.bar.style.width = `${pct}%`;
|
||||||
|
const hue = Math.floor(pct * 1.2); // 0..120
|
||||||
|
ui.bar.style.backgroundColor = `hsl(${hue}, 80%, 50%)`;
|
||||||
|
|
||||||
|
// slider (only meaningful when not connected; keep in sync anyway)
|
||||||
|
const rawForSlider = info.inverted ? (info.min + info.max) - value : value;
|
||||||
|
if (!isConnected) ui.slider.value = String(clamp(Math.round(rawForSlider), RAW_MIN, RAW_MAX));
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Serial I/O --------------------
|
||||||
|
async function readSerialData() {
|
||||||
|
while (port?.readable && keepReading) {
|
||||||
|
reader = port.readable.getReader();
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const { value, done } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
if (value) processData(decoder.decode(value));
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error reading:', err);
|
||||||
|
addLogMessage(`Error: ${err.message}`);
|
||||||
|
break;
|
||||||
|
} finally {
|
||||||
|
reader.releaseLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function processData(chunk) {
|
||||||
|
inputBuffer += chunk;
|
||||||
|
let idx;
|
||||||
|
while ((idx = inputBuffer.indexOf('\n')) !== -1) {
|
||||||
|
const line = inputBuffer.slice(0, idx).trim();
|
||||||
|
inputBuffer = inputBuffer.slice(idx + 1);
|
||||||
|
|
||||||
|
const vals = line.split(/\s+/).map(v => parseInt(v, 10));
|
||||||
|
if (vals.length === MAX_JOINTS && vals.every(v => Number.isFinite(v))) {
|
||||||
|
for (let i = 0; i < MAX_JOINTS; i++) {
|
||||||
|
const info = fingerJointMap[i];
|
||||||
|
if (!info) continue;
|
||||||
|
|
||||||
|
let rawValue = vals[i];
|
||||||
|
|
||||||
|
// Update calibration tracking
|
||||||
|
if (calibrationEnabled) {
|
||||||
|
observedMin[i] = Math.min(observedMin[i], rawValue);
|
||||||
|
observedMax[i] = Math.max(observedMax[i], rawValue);
|
||||||
|
|
||||||
|
// Update calibration display
|
||||||
|
const ui = uiRefs[i];
|
||||||
|
if (ui && ui.calibStatus) {
|
||||||
|
if (observedMin[i] !== Infinity && observedMax[i] !== -Infinity) {
|
||||||
|
ui.calibStatus.textContent = `Range: ${observedMin[i]}-${observedMax[i]}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remap observed range to target range
|
||||||
|
if (observedMin[i] !== Infinity && observedMax[i] !== -Infinity && observedMax[i] > observedMin[i]) {
|
||||||
|
const observedRange = observedMax[i] - observedMin[i];
|
||||||
|
const targetRange = info.max - info.min;
|
||||||
|
const normalizedValue = (rawValue - observedMin[i]) / observedRange;
|
||||||
|
rawValue = info.min + (normalizedValue * targetRange);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let v = clamp(rawValue, info.min, info.max);
|
||||||
|
if (info.inverted) v = (info.min + info.max) - v;
|
||||||
|
jointValues[i] = v;
|
||||||
|
updateJointDisplay(i, v);
|
||||||
|
}
|
||||||
|
updateHandModel();
|
||||||
|
} else {
|
||||||
|
addLogMessage(`Received: ${line}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function connectToDevice() {
|
||||||
|
try {
|
||||||
|
port = await navigator.serial.requestPort();
|
||||||
|
const baudRate = parseInt(baudRateSelect.value, 10) || 115200;
|
||||||
|
await port.open({ baudRate });
|
||||||
|
|
||||||
|
keepReading = true;
|
||||||
|
setConnectedUI(true);
|
||||||
|
|
||||||
|
statusIndicator.textContent = 'Status: Connected';
|
||||||
|
statusIndicator.className = 'status connected';
|
||||||
|
connectButton.disabled = true;
|
||||||
|
disconnectButton.disabled = false;
|
||||||
|
baudRateSelect.disabled = true;
|
||||||
|
|
||||||
|
addLogMessage(`Connected at ${baudRate} baud`);
|
||||||
|
readSerialData();
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Connect error:', e);
|
||||||
|
addLogMessage(`Connection error: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function disconnectFromDevice() {
|
||||||
|
try {
|
||||||
|
keepReading = false;
|
||||||
|
if (reader) {
|
||||||
|
try { reader.cancel(); } catch {}
|
||||||
|
}
|
||||||
|
if (port) {
|
||||||
|
await port.close();
|
||||||
|
port = null;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Disconnect error:', e);
|
||||||
|
addLogMessage(`Disconnection error: ${e.message}`);
|
||||||
|
} finally {
|
||||||
|
setConnectedUI(false);
|
||||||
|
statusIndicator.textContent = 'Status: Disconnected';
|
||||||
|
statusIndicator.className = 'status disconnected';
|
||||||
|
connectButton.disabled = false;
|
||||||
|
disconnectButton.disabled = true;
|
||||||
|
baudRateSelect.disabled = false;
|
||||||
|
addLogMessage('Disconnected');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Three.js Scene --------------------
|
||||||
|
function initThreeJS() {
|
||||||
|
scene = new THREE.Scene();
|
||||||
|
scene.background = new THREE.Color(0xf0f0f0);
|
||||||
|
|
||||||
|
camera = new THREE.PerspectiveCamera(
|
||||||
|
75,
|
||||||
|
canvasContainer.clientWidth / canvasContainer.clientHeight,
|
||||||
|
0.1, 1000
|
||||||
|
);
|
||||||
|
camera.position.set(0, 15, 15);
|
||||||
|
camera.lookAt(0, 0, 0);
|
||||||
|
|
||||||
|
renderer = new THREE.WebGLRenderer({ antialias: true });
|
||||||
|
renderer.setSize(canvasContainer.clientWidth, canvasContainer.clientHeight);
|
||||||
|
renderer.setPixelRatio(window.devicePixelRatio);
|
||||||
|
canvasContainer.appendChild(renderer.domElement);
|
||||||
|
|
||||||
|
controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||||
|
controls.enableDamping = true;
|
||||||
|
controls.dampingFactor = 0.25;
|
||||||
|
|
||||||
|
const ambientLight = new THREE.AmbientLight(0x404040);
|
||||||
|
scene.add(ambientLight);
|
||||||
|
const dir1 = new THREE.DirectionalLight(0xffffff, 0.5);
|
||||||
|
dir1.position.set(1, 1, 1);
|
||||||
|
scene.add(dir1);
|
||||||
|
const dir2 = new THREE.DirectionalLight(0xffffff, 0.3);
|
||||||
|
dir2.position.set(-1, 1, -1);
|
||||||
|
scene.add(dir2);
|
||||||
|
|
||||||
|
const gridHelper = new THREE.GridHelper(20, 20);
|
||||||
|
scene.add(gridHelper);
|
||||||
|
|
||||||
|
createHandModel();
|
||||||
|
window.addEventListener('resize', onWindowResize);
|
||||||
|
animate();
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHandModel() {
|
||||||
|
const palmMaterial = new THREE.MeshPhongMaterial({ color: 0xf5c396 });
|
||||||
|
const fingerMaterial = new THREE.MeshPhongMaterial({ color: 0xf5c396 });
|
||||||
|
const jointMaterial = new THREE.MeshPhongMaterial({ color: 0xe3a977 });
|
||||||
|
|
||||||
|
const palmGeometry = new THREE.BoxGeometry(7, 1, 8);
|
||||||
|
hand.palm = new THREE.Mesh(palmGeometry, palmMaterial);
|
||||||
|
hand.palm.position.set(0, 0, 0);
|
||||||
|
hand.palm.rotation.x = Math.PI / 2; // hand vertical, palm facing forward
|
||||||
|
scene.add(hand.palm);
|
||||||
|
|
||||||
|
const fingerWidth = 1, fingerHeight = 0.8;
|
||||||
|
const fingerSegmentLengths = [3, 2, 1.5];
|
||||||
|
const thumbSegmentLengths = [2, 2, 1.5];
|
||||||
|
|
||||||
|
const fingerBasePositions = [
|
||||||
|
[ 3, 0, -2], // Thumb
|
||||||
|
[ 1.5,-0.5,-4], // Index
|
||||||
|
[ 0, -0.5,-4], // Middle
|
||||||
|
[-1.5,-0.5,-4], // Ring
|
||||||
|
[-3, -0.5,-4], // Pinky
|
||||||
|
];
|
||||||
|
const fingerBaseRot = [
|
||||||
|
{ x:0, y:-Math.PI/3, z: Math.PI/3 }, // Thumb
|
||||||
|
{ x:0, y:-Math.PI/48, z: 0 },
|
||||||
|
{ x:0, y: Math.PI/48, z: 0 },
|
||||||
|
{ x:0, y: Math.PI/32, z: 0 },
|
||||||
|
{ x:0, y: Math.PI/24, z: 0 }
|
||||||
|
];
|
||||||
|
|
||||||
|
for (let fIdx = 0; fIdx < 5; fIdx++) {
|
||||||
|
const finger = { name:['Thumb','Index','Middle','Ring','Pinky'][fIdx], segments:[], joints:[] };
|
||||||
|
const isThumb = fIdx === 0;
|
||||||
|
const segLens = isThumb ? thumbSegmentLengths : fingerSegmentLengths;
|
||||||
|
|
||||||
|
finger.group = new THREE.Group();
|
||||||
|
finger.group.position.set(...fingerBasePositions[fIdx]);
|
||||||
|
finger.group.rotation.x = fingerBaseRot[fIdx].x;
|
||||||
|
finger.group.rotation.y = fingerBaseRot[fIdx].y;
|
||||||
|
finger.group.rotation.z = fingerBaseRot[fIdx].z;
|
||||||
|
finger.group.userData.baseRot = {
|
||||||
|
x:finger.group.rotation.x,
|
||||||
|
y:finger.group.rotation.y,
|
||||||
|
z:finger.group.rotation.z
|
||||||
|
};
|
||||||
|
hand.palm.add(finger.group);
|
||||||
|
|
||||||
|
let parent = finger.group;
|
||||||
|
for (let s = 0; s < segLens.length; s++) {
|
||||||
|
const segGroup = new THREE.Group();
|
||||||
|
|
||||||
|
const jGeom = new THREE.SphereGeometry(fingerWidth * 0.6, 8, 8);
|
||||||
|
const joint = new THREE.Mesh(jGeom, jointMaterial);
|
||||||
|
segGroup.add(joint);
|
||||||
|
|
||||||
|
const segGeom = new THREE.BoxGeometry(fingerWidth, fingerHeight, segLens[s]);
|
||||||
|
const seg = new THREE.Mesh(segGeom, fingerMaterial);
|
||||||
|
seg.position.z = -segLens[s] / 2;
|
||||||
|
segGroup.add(seg);
|
||||||
|
|
||||||
|
parent.add(segGroup);
|
||||||
|
|
||||||
|
finger.segments.push(segGroup);
|
||||||
|
finger.joints.push(joint);
|
||||||
|
|
||||||
|
if (s < segLens.length - 1) {
|
||||||
|
const connector = new THREE.Group();
|
||||||
|
connector.position.z = -segLens[s];
|
||||||
|
segGroup.add(connector);
|
||||||
|
parent = connector;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
hand.fingers.push(finger);
|
||||||
|
}
|
||||||
|
|
||||||
|
addFingerLabels();
|
||||||
|
addHandLabel();
|
||||||
|
}
|
||||||
|
|
||||||
|
function addFingerLabels() {
|
||||||
|
const names = ['Thumb','Index','Middle','Ring','Pinky'];
|
||||||
|
for (let i = 0; i < hand.fingers.length; i++) {
|
||||||
|
const finger = hand.fingers[i];
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
canvas.width = 128; canvas.height = 32;
|
||||||
|
ctx.fillStyle = '#ffffff'; ctx.fillRect(0,0,canvas.width,canvas.height);
|
||||||
|
ctx.font = 'bold 16px Arial';
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
|
||||||
|
ctx.fillText(names[i], canvas.width/2, canvas.height/2);
|
||||||
|
|
||||||
|
const texture = new THREE.CanvasTexture(canvas);
|
||||||
|
const geom = new THREE.PlaneGeometry(2, 0.5);
|
||||||
|
const mat = new THREE.MeshBasicMaterial({ map:texture, transparent:true, side:THREE.DoubleSide });
|
||||||
|
const label = new THREE.Mesh(geom, mat);
|
||||||
|
label.position.set(0, -1.5, -2);
|
||||||
|
label.rotation.x = Math.PI / 2;
|
||||||
|
finger.group.add(label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function addHandLabel() {
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
canvas.width = 256; canvas.height = 64;
|
||||||
|
ctx.fillStyle = '#ffffff'; ctx.fillRect(0,0,canvas.width,canvas.height);
|
||||||
|
ctx.font = 'bold 24px Arial';
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
|
||||||
|
ctx.fillText('RIGHT HAND (VERTICAL)', canvas.width/2, canvas.height/2);
|
||||||
|
|
||||||
|
const texture = new THREE.CanvasTexture(canvas);
|
||||||
|
const geom = new THREE.PlaneGeometry(7, 1.75);
|
||||||
|
const mat = new THREE.MeshBasicMaterial({ map:texture, transparent:true, side:THREE.DoubleSide });
|
||||||
|
const label = new THREE.Mesh(geom, mat);
|
||||||
|
label.position.set(0, -2, 0);
|
||||||
|
label.rotation.x = Math.PI / 2;
|
||||||
|
scene.add(label);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateHandModel() {
|
||||||
|
for (let i = 0; i < MAX_JOINTS; i++) {
|
||||||
|
const info = fingerJointMap[i];
|
||||||
|
if (!info) continue;
|
||||||
|
const { finger, joint, type, min, max, angleMin, angleMax } = info;
|
||||||
|
const raw = jointValues[i];
|
||||||
|
const f = hand.fingers[finger];
|
||||||
|
if (!f) continue;
|
||||||
|
|
||||||
|
const center = (min + max) / 2;
|
||||||
|
let angle = 0;
|
||||||
|
|
||||||
|
if (type.includes('ABDUCTION')) {
|
||||||
|
// symmetric around neutral
|
||||||
|
const k = clamp((raw - center) / ((max - min) / 2), -1, 1);
|
||||||
|
angle = angleMin + (k + 1) * 0.5 * (angleMax - angleMin);
|
||||||
|
|
||||||
|
const base = f.group.userData.baseRot || {x:0,y:0,z:0};
|
||||||
|
if (finger === 0 && joint === 0) {
|
||||||
|
// Thumb: abduction about Z (toward/away from palm)
|
||||||
|
f.group.rotation.z = base.z + angle;
|
||||||
|
} else {
|
||||||
|
// Other fingers: side-to-side about Y
|
||||||
|
f.group.rotation.y = base.y + angle;
|
||||||
|
}
|
||||||
|
} else if (type.includes('FLEXION')) {
|
||||||
|
const isThumb = finger === 0;
|
||||||
|
const isMCP = type === 'MCP_FLEXION';
|
||||||
|
const isPIP = type === 'PIP_FLEXION';
|
||||||
|
const positiveOnly = (isThumb && (type === 'MCP_FLEXION' || type === 'IP_FLEXION')) || (!isThumb && isPIP);
|
||||||
|
|
||||||
|
if (positiveOnly) {
|
||||||
|
const t = raw <= center ? 0 : invLerp(center, max, raw); // 0..1
|
||||||
|
angle = angleMin + t * (angleMax - angleMin); // 0..+limit
|
||||||
|
} else {
|
||||||
|
const k = clamp((raw - center) / ((max - min) / 2), -1, 1);
|
||||||
|
angle = angleMin + (k + 1) * 0.5 * (angleMax - angleMin);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isMCP) {
|
||||||
|
// MCP flexion applies to the finger base group (same as abduction)
|
||||||
|
const base = f.group.userData.baseRot || {x:0,y:0,z:0};
|
||||||
|
f.group.rotation.x = base.x + angle;
|
||||||
|
} else if (f.segments[joint]) {
|
||||||
|
// PIP/DIP/IP flexion applies to individual segments
|
||||||
|
f.segments[joint].rotation.x = angle;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Render Loop --------------------
|
||||||
|
function onWindowResize() {
|
||||||
|
camera.aspect = canvasContainer.clientWidth / canvasContainer.clientHeight;
|
||||||
|
camera.updateProjectionMatrix();
|
||||||
|
renderer.setSize(canvasContainer.clientWidth, canvasContainer.clientHeight);
|
||||||
|
}
|
||||||
|
|
||||||
|
function animate() {
|
||||||
|
requestAnimationFrame(animate);
|
||||||
|
controls.update();
|
||||||
|
renderer.render(scene, camera);
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Misc UI --------------------
|
||||||
|
function addLogMessage(msg) {
|
||||||
|
const el = document.createElement('div');
|
||||||
|
el.textContent = msg;
|
||||||
|
logContainer.appendChild(el);
|
||||||
|
logContainer.scrollTop = logContainer.scrollHeight;
|
||||||
|
while (logContainer.children.length > 100) {
|
||||||
|
logContainer.removeChild(logContainer.firstChild);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Camera view controls
|
||||||
|
frontViewBtn?.addEventListener('click', () => { camera.position.set(0, 0, 20); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
sideViewBtn?.addEventListener('click', () => { camera.position.set(20, 0, 0); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
topViewBtn?.addEventListener('click', () => { camera.position.set(0, 20, 0); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
resetViewBtn?.addEventListener('click', () => { camera.position.set(10,10,10); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
|
||||||
|
// Serial connect buttons
|
||||||
|
connectButton?.addEventListener('click', connectToDevice);
|
||||||
|
disconnectButton?.addEventListener('click', disconnectFromDevice);
|
||||||
|
|
||||||
|
// Web Serial support check
|
||||||
|
if (!navigator.serial) {
|
||||||
|
statusIndicator.textContent = 'Status: Web Serial API not supported in this browser';
|
||||||
|
connectButton.disabled = true;
|
||||||
|
addLogMessage('ERROR: Web Serial API is not supported in this browser. Try Chrome or Edge.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Boot --------------------
|
||||||
|
initThreeJS();
|
||||||
|
initializeJointElements();
|
||||||
|
|
||||||
|
// -------------------- Styles (inline) --------------------
|
||||||
|
const styleElement = document.createElement('style');
|
||||||
|
styleElement.textContent = `
|
||||||
|
.joint-info { border-bottom: 1px solid #eee; padding: 8px 0; }
|
||||||
|
.joint-name { font-weight: 600; margin-bottom: 4px; }
|
||||||
|
.joint-value { font-size: 12px; color: #333; margin-bottom: 4px; }
|
||||||
|
.bar-container { width: 100%; height: 8px; background: #ddd; border-radius: 4px; overflow: hidden; }
|
||||||
|
.bar { height: 100%; width: 0%; background: #4caf50; }
|
||||||
|
.joint-slider { width: 100%; margin: 6px 0; }
|
||||||
|
.invert-toggle { display: inline-flex; align-items: center; gap: 6px; margin-top: 4px; font-size: 12px; color: #555; }
|
||||||
|
.limits-row { display: flex; align-items: center; gap: 6px; margin-top: 6px; flex-wrap: wrap; }
|
||||||
|
.limit-label { font-size: 11px; color: #666; }
|
||||||
|
.limit-num { width: 60px; }
|
||||||
|
.calib-row { display: flex; align-items: center; gap: 8px; margin-top: 4px; }
|
||||||
|
.calib-btn { padding: 2px 6px; font-size: 11px; background: #f44336; color: white; border: none; border-radius: 3px; cursor: pointer; }
|
||||||
|
.calib-btn:hover { background: #d32f2f; }
|
||||||
|
.calib-status { font-size: 11px; color: #666; }
|
||||||
|
.status.connected { color: #0a0; }
|
||||||
|
.status.disconnected { color: #a00; }
|
||||||
|
`;
|
||||||
|
document.head.appendChild(styleElement);
|
||||||
@@ -825,8 +825,6 @@ class LeRobotDataset(torch.utils.data.Dataset):
|
|||||||
"""
|
"""
|
||||||
if not episode_data:
|
if not episode_data:
|
||||||
episode_buffer = self.episode_buffer
|
episode_buffer = self.episode_buffer
|
||||||
else:
|
|
||||||
episode_buffer = episode_data
|
|
||||||
|
|
||||||
validate_episode_buffer(episode_buffer, self.meta.total_episodes, self.features)
|
validate_episode_buffer(episode_buffer, self.meta.total_episodes, self.features)
|
||||||
|
|
||||||
|
|||||||
@@ -107,8 +107,6 @@ X_SERIES_ENCODINGS_TABLE = {
|
|||||||
"Goal_PWM": X_SERIES_CONTROL_TABLE["Goal_PWM"][1],
|
"Goal_PWM": X_SERIES_CONTROL_TABLE["Goal_PWM"][1],
|
||||||
"Goal_Current": X_SERIES_CONTROL_TABLE["Goal_Current"][1],
|
"Goal_Current": X_SERIES_CONTROL_TABLE["Goal_Current"][1],
|
||||||
"Goal_Velocity": X_SERIES_CONTROL_TABLE["Goal_Velocity"][1],
|
"Goal_Velocity": X_SERIES_CONTROL_TABLE["Goal_Velocity"][1],
|
||||||
"Goal_Position": X_SERIES_CONTROL_TABLE["Goal_Position"][1],
|
|
||||||
"Present_Position": X_SERIES_CONTROL_TABLE["Present_Position"][1],
|
|
||||||
"Present_PWM": X_SERIES_CONTROL_TABLE["Present_PWM"][1],
|
"Present_PWM": X_SERIES_CONTROL_TABLE["Present_PWM"][1],
|
||||||
"Present_Current": X_SERIES_CONTROL_TABLE["Present_Current"][1],
|
"Present_Current": X_SERIES_CONTROL_TABLE["Present_Current"][1],
|
||||||
"Present_Velocity": X_SERIES_CONTROL_TABLE["Present_Velocity"][1],
|
"Present_Velocity": X_SERIES_CONTROL_TABLE["Present_Velocity"][1],
|
||||||
|
|||||||
@@ -161,11 +161,6 @@ class SO100Follower(Robot):
|
|||||||
self.bus.write("I_Coefficient", motor, 0)
|
self.bus.write("I_Coefficient", motor, 0)
|
||||||
self.bus.write("D_Coefficient", motor, 32)
|
self.bus.write("D_Coefficient", motor, 32)
|
||||||
|
|
||||||
if motor == "gripper":
|
|
||||||
self.bus.write("Max_Torque_Limit", motor, 500) # 50% of max torque to avoid burnout
|
|
||||||
self.bus.write("Protection_Current", motor, 250) # 50% of max current to avoid burnout
|
|
||||||
self.bus.write("Overload_Torque", motor, 25) # 25% torque when overloaded
|
|
||||||
|
|
||||||
def setup_motors(self) -> None:
|
def setup_motors(self) -> None:
|
||||||
for motor in reversed(self.bus.motors):
|
for motor in reversed(self.bus.motors):
|
||||||
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
||||||
|
|||||||
@@ -157,13 +157,6 @@ class SO101Follower(Robot):
|
|||||||
self.bus.write("I_Coefficient", motor, 0)
|
self.bus.write("I_Coefficient", motor, 0)
|
||||||
self.bus.write("D_Coefficient", motor, 32)
|
self.bus.write("D_Coefficient", motor, 32)
|
||||||
|
|
||||||
if motor == "gripper":
|
|
||||||
self.bus.write(
|
|
||||||
"Max_Torque_Limit", motor, 500
|
|
||||||
) # 50% of the max torque limit to avoid burnout
|
|
||||||
self.bus.write("Protection_Current", motor, 250) # 50% of max current to avoid burnout
|
|
||||||
self.bus.write("Overload_Torque", motor, 25) # 25% torque when overloaded
|
|
||||||
|
|
||||||
def setup_motors(self) -> None:
|
def setup_motors(self) -> None:
|
||||||
for motor in reversed(self.bus.motors):
|
for motor in reversed(self.bus.motors):
|
||||||
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
input(f"Connect the controller board to the '{motor}' motor only and press enter.")
|
||||||
|
|||||||
@@ -243,11 +243,7 @@ def eval_policy(
|
|||||||
if max_episodes_rendered > 0 and not videos_dir:
|
if max_episodes_rendered > 0 and not videos_dir:
|
||||||
raise ValueError("If max_episodes_rendered > 0, videos_dir must be provided.")
|
raise ValueError("If max_episodes_rendered > 0, videos_dir must be provided.")
|
||||||
|
|
||||||
# Handle accelerate-wrapped models by unwrapping them
|
if not isinstance(policy, PreTrainedPolicy):
|
||||||
if hasattr(policy, 'module') and isinstance(policy.module, PreTrainedPolicy):
|
|
||||||
# This is likely an accelerate-wrapped model (DistributedDataParallel)
|
|
||||||
policy = policy.module
|
|
||||||
elif not isinstance(policy, PreTrainedPolicy):
|
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Policy of type 'PreTrainedPolicy' is expected, but type '{type(policy)}' was provided."
|
f"Policy of type 'PreTrainedPolicy' is expected, but type '{type(policy)}' was provided."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -302,6 +302,11 @@ class RobotClient:
|
|||||||
|
|
||||||
self.logger.debug(f"Current latest action: {latest_action}")
|
self.logger.debug(f"Current latest action: {latest_action}")
|
||||||
|
|
||||||
|
# Get queue state before changes
|
||||||
|
old_size, old_timesteps = self._inspect_action_queue()
|
||||||
|
if not old_timesteps:
|
||||||
|
old_timesteps = [latest_action] # queue was empty
|
||||||
|
|
||||||
# Get queue state before changes
|
# Get queue state before changes
|
||||||
old_size, old_timesteps = self._inspect_action_queue()
|
old_size, old_timesteps = self._inspect_action_queue()
|
||||||
if not old_timesteps:
|
if not old_timesteps:
|
||||||
|
|||||||
@@ -16,7 +16,6 @@
|
|||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from contextlib import nullcontext
|
from contextlib import nullcontext
|
||||||
from functools import partial
|
|
||||||
from pprint import pformat
|
from pprint import pformat
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -24,8 +23,6 @@ import torch
|
|||||||
from termcolor import colored
|
from termcolor import colored
|
||||||
from torch.amp import GradScaler
|
from torch.amp import GradScaler
|
||||||
from torch.optim import Optimizer
|
from torch.optim import Optimizer
|
||||||
import os
|
|
||||||
from datetime import timedelta
|
|
||||||
|
|
||||||
from lerobot.configs import parser
|
from lerobot.configs import parser
|
||||||
from lerobot.configs.train import TrainPipelineConfig
|
from lerobot.configs.train import TrainPipelineConfig
|
||||||
@@ -55,8 +52,6 @@ from lerobot.utils.utils import (
|
|||||||
)
|
)
|
||||||
from lerobot.utils.wandb_utils import WandBLogger
|
from lerobot.utils.wandb_utils import WandBLogger
|
||||||
|
|
||||||
def is_launched_with_accelerate() -> bool:
|
|
||||||
return "ACCELERATE_MIXED_PRECISION" in os.environ
|
|
||||||
|
|
||||||
def update_policy(
|
def update_policy(
|
||||||
train_metrics: MetricsTracker,
|
train_metrics: MetricsTracker,
|
||||||
@@ -64,65 +59,36 @@ def update_policy(
|
|||||||
batch: Any,
|
batch: Any,
|
||||||
optimizer: Optimizer,
|
optimizer: Optimizer,
|
||||||
grad_clip_norm: float,
|
grad_clip_norm: float,
|
||||||
grad_scaler: GradScaler | None,
|
grad_scaler: GradScaler,
|
||||||
lr_scheduler=None,
|
lr_scheduler=None,
|
||||||
use_amp: bool = False,
|
use_amp: bool = False,
|
||||||
lock=None,
|
lock=None,
|
||||||
accelerator=None,
|
|
||||||
) -> tuple[MetricsTracker, dict]:
|
) -> tuple[MetricsTracker, dict]:
|
||||||
start_time = time.perf_counter()
|
start_time = time.perf_counter()
|
||||||
device = get_device_from_parameters(policy)
|
device = get_device_from_parameters(policy)
|
||||||
policy.train()
|
policy.train()
|
||||||
|
with torch.autocast(device_type=device.type) if use_amp else nullcontext():
|
||||||
grad_norm = 0.0 # Initialize grad_norm to avoid undefined variable
|
loss, output_dict = policy.forward(batch)
|
||||||
|
|
||||||
if accelerator:
|
|
||||||
with accelerator.accumulate(policy):
|
|
||||||
with torch.autocast(device_type=device.type) if use_amp else nullcontext():
|
|
||||||
loss, output_dict = policy.forward(batch)
|
|
||||||
# TODO(rcadene): policy.unnormalize_outputs(out_dict)
|
|
||||||
accelerator.backward(loss)
|
|
||||||
if accelerator.sync_gradients:
|
|
||||||
grad_norm = torch.nn.utils.clip_grad_norm_(
|
|
||||||
policy.parameters(),
|
|
||||||
grad_clip_norm,
|
|
||||||
error_if_nonfinite=False,
|
|
||||||
)
|
|
||||||
optimizer.step()
|
|
||||||
optimizer.zero_grad()
|
|
||||||
else:
|
|
||||||
# Standard training loop without accelerate
|
|
||||||
with torch.autocast(device_type=device.type) if use_amp else nullcontext():
|
|
||||||
loss, output_dict = policy.forward(batch)
|
|
||||||
# TODO(rcadene): policy.unnormalize_outputs(out_dict)
|
# TODO(rcadene): policy.unnormalize_outputs(out_dict)
|
||||||
|
grad_scaler.scale(loss).backward()
|
||||||
|
|
||||||
if grad_scaler is not None:
|
# Unscale the gradient of the optimizer's assigned params in-place **prior to gradient clipping**.
|
||||||
grad_scaler.scale(loss).backward()
|
grad_scaler.unscale_(optimizer)
|
||||||
# Unscale the gradient of the optimizer's assigned params in-place **prior to gradient clipping**.
|
|
||||||
grad_scaler.unscale_(optimizer)
|
|
||||||
grad_norm = torch.nn.utils.clip_grad_norm_(
|
|
||||||
policy.parameters(),
|
|
||||||
grad_clip_norm,
|
|
||||||
error_if_nonfinite=False,
|
|
||||||
)
|
|
||||||
# Optimizer's gradients are already unscaled, so scaler.step does not unscale them,
|
|
||||||
# although it still skips optimizer.step() if the gradients contain infs or NaNs.
|
|
||||||
with lock if lock is not None else nullcontext():
|
|
||||||
grad_scaler.step(optimizer)
|
|
||||||
# Updates the scale for next iteration.
|
|
||||||
grad_scaler.update()
|
|
||||||
else:
|
|
||||||
# Without GradScaler (fallback)
|
|
||||||
loss.backward()
|
|
||||||
grad_norm = torch.nn.utils.clip_grad_norm_(
|
|
||||||
policy.parameters(),
|
|
||||||
grad_clip_norm,
|
|
||||||
error_if_nonfinite=False,
|
|
||||||
)
|
|
||||||
with lock if lock is not None else nullcontext():
|
|
||||||
optimizer.step()
|
|
||||||
|
|
||||||
optimizer.zero_grad()
|
grad_norm = torch.nn.utils.clip_grad_norm_(
|
||||||
|
policy.parameters(),
|
||||||
|
grad_clip_norm,
|
||||||
|
error_if_nonfinite=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Optimizer's gradients are already unscaled, so scaler.step does not unscale them,
|
||||||
|
# although it still skips optimizer.step() if the gradients contain infs or NaNs.
|
||||||
|
with lock if lock is not None else nullcontext():
|
||||||
|
grad_scaler.step(optimizer)
|
||||||
|
# Updates the scale for next iteration.
|
||||||
|
grad_scaler.update()
|
||||||
|
|
||||||
|
optimizer.zero_grad()
|
||||||
|
|
||||||
# Step through pytorch scheduler at every batch instead of epoch
|
# Step through pytorch scheduler at every batch instead of epoch
|
||||||
if lr_scheduler is not None:
|
if lr_scheduler is not None:
|
||||||
@@ -133,7 +99,7 @@ def update_policy(
|
|||||||
policy.update()
|
policy.update()
|
||||||
|
|
||||||
train_metrics.loss = loss.item()
|
train_metrics.loss = loss.item()
|
||||||
train_metrics.grad_norm = grad_norm.item() if isinstance(grad_norm, torch.Tensor) else grad_norm
|
train_metrics.grad_norm = grad_norm.item()
|
||||||
train_metrics.lr = optimizer.param_groups[0]["lr"]
|
train_metrics.lr = optimizer.param_groups[0]["lr"]
|
||||||
train_metrics.update_s = time.perf_counter() - start_time
|
train_metrics.update_s = time.perf_counter() - start_time
|
||||||
return train_metrics, output_dict
|
return train_metrics, output_dict
|
||||||
@@ -142,33 +108,8 @@ def update_policy(
|
|||||||
@parser.wrap()
|
@parser.wrap()
|
||||||
def train(cfg: TrainPipelineConfig):
|
def train(cfg: TrainPipelineConfig):
|
||||||
cfg.validate()
|
cfg.validate()
|
||||||
|
|
||||||
accelerator = None
|
|
||||||
if is_launched_with_accelerate():
|
|
||||||
import accelerate
|
|
||||||
|
|
||||||
# For example pi0 has unused params (last llm block)
|
|
||||||
from accelerate import DistributedDataParallelKwargs
|
|
||||||
ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)
|
|
||||||
# accelerator = accelerate.Accelerator(step_scheduler_with_optimizer=False, kwargs_handlers=[ddp_kwargs])
|
|
||||||
from accelerate import InitProcessGroupKwargs
|
|
||||||
# Set NCCL timeout (default 30 minutes = 1800 seconds)
|
|
||||||
nccl_timeout = getattr(cfg, 'nccl_timeout', 1800)
|
|
||||||
ddp_init_kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=nccl_timeout)) # FIXME(mshukor): allow user to set timeout. This should be longer than the evaluation time
|
|
||||||
# Set gradient accumulation steps (default 1)
|
|
||||||
gradient_accumulation_steps = getattr(cfg, 'gradient_accumulation_steps', 1)
|
|
||||||
accelerator = accelerate.Accelerator(step_scheduler_with_optimizer=False, gradient_accumulation_steps=gradient_accumulation_steps, kwargs_handlers=[ddp_init_kwargs, ddp_kwargs])
|
|
||||||
if accelerator is not None and not accelerator.is_main_process:
|
|
||||||
# Disable duplicate logging on non-main processes
|
|
||||||
logging.info(f"Setting logging level on non-main process {accelerator.process_index} to WARNING.")
|
|
||||||
logging.getLogger().setLevel(logging.WARNING)
|
|
||||||
|
|
||||||
logging.info(pformat(cfg.to_dict()))
|
logging.info(pformat(cfg.to_dict()))
|
||||||
|
|
||||||
if accelerator and not accelerator.is_main_process:
|
|
||||||
# Disable logging on non-main processes.
|
|
||||||
cfg.wandb.enable = False
|
|
||||||
|
|
||||||
if cfg.wandb.enable and cfg.wandb.project:
|
if cfg.wandb.enable and cfg.wandb.project:
|
||||||
wandb_logger = WandBLogger(cfg)
|
wandb_logger = WandBLogger(cfg)
|
||||||
else:
|
else:
|
||||||
@@ -202,8 +143,7 @@ def train(cfg: TrainPipelineConfig):
|
|||||||
|
|
||||||
logging.info("Creating optimizer and scheduler")
|
logging.info("Creating optimizer and scheduler")
|
||||||
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy)
|
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy)
|
||||||
# Only use GradScaler when not using accelerate (accelerate handles mixed precision internally)
|
grad_scaler = GradScaler(device.type, enabled=cfg.policy.use_amp)
|
||||||
grad_scaler = None if accelerator else GradScaler(device.type, enabled=cfg.policy.use_amp)
|
|
||||||
|
|
||||||
step = 0 # number of policy updates (forward + backward + optim)
|
step = 0 # number of policy updates (forward + backward + optim)
|
||||||
|
|
||||||
@@ -245,11 +185,6 @@ def train(cfg: TrainPipelineConfig):
|
|||||||
)
|
)
|
||||||
dl_iter = cycle(dataloader)
|
dl_iter = cycle(dataloader)
|
||||||
|
|
||||||
# Prepare models for accelerate if using multi-GPU
|
|
||||||
if accelerator:
|
|
||||||
policy, optimizer, dataloader = accelerator.prepare(policy, optimizer, dataloader)
|
|
||||||
dl_iter = cycle(dataloader)
|
|
||||||
|
|
||||||
policy.train()
|
policy.train()
|
||||||
|
|
||||||
train_metrics = {
|
train_metrics = {
|
||||||
@@ -270,10 +205,9 @@ def train(cfg: TrainPipelineConfig):
|
|||||||
batch = next(dl_iter)
|
batch = next(dl_iter)
|
||||||
train_tracker.dataloading_s = time.perf_counter() - start_time
|
train_tracker.dataloading_s = time.perf_counter() - start_time
|
||||||
|
|
||||||
if not accelerator:
|
for key in batch:
|
||||||
for key in batch:
|
if isinstance(batch[key], torch.Tensor):
|
||||||
if isinstance(batch[key], torch.Tensor):
|
batch[key] = batch[key].to(device, non_blocking=device.type == "cuda")
|
||||||
batch[key] = batch[key].to(device, non_blocking=device.type == "cuda")
|
|
||||||
|
|
||||||
train_tracker, output_dict = update_policy(
|
train_tracker, output_dict = update_policy(
|
||||||
train_tracker,
|
train_tracker,
|
||||||
@@ -284,7 +218,6 @@ def train(cfg: TrainPipelineConfig):
|
|||||||
grad_scaler=grad_scaler,
|
grad_scaler=grad_scaler,
|
||||||
lr_scheduler=lr_scheduler,
|
lr_scheduler=lr_scheduler,
|
||||||
use_amp=cfg.policy.use_amp,
|
use_amp=cfg.policy.use_amp,
|
||||||
accelerator=accelerator,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Note: eval and checkpoint happens *after* the `step`th training update has completed, so we
|
# Note: eval and checkpoint happens *after* the `step`th training update has completed, so we
|
||||||
@@ -304,17 +237,15 @@ def train(cfg: TrainPipelineConfig):
|
|||||||
wandb_logger.log_dict(wandb_log_dict, step)
|
wandb_logger.log_dict(wandb_log_dict, step)
|
||||||
train_tracker.reset_averages()
|
train_tracker.reset_averages()
|
||||||
|
|
||||||
if cfg.save_checkpoint and is_saving_step and (not accelerator or accelerator.is_main_process):
|
if cfg.save_checkpoint and is_saving_step:
|
||||||
logging.info(f"Checkpoint policy after step {step}")
|
logging.info(f"Checkpoint policy after step {step}")
|
||||||
checkpoint_dir = get_step_checkpoint_dir(cfg.output_dir, cfg.steps, step)
|
checkpoint_dir = get_step_checkpoint_dir(cfg.output_dir, cfg.steps, step)
|
||||||
# Unwrap model for accelerate
|
save_checkpoint(checkpoint_dir, step, cfg, policy, optimizer, lr_scheduler)
|
||||||
policy_to_save = accelerator.unwrap_model(policy) if accelerator else policy
|
|
||||||
save_checkpoint(checkpoint_dir, step, cfg, policy_to_save, optimizer, lr_scheduler)
|
|
||||||
update_last_checkpoint(checkpoint_dir)
|
update_last_checkpoint(checkpoint_dir)
|
||||||
if wandb_logger:
|
if wandb_logger:
|
||||||
wandb_logger.log_policy(checkpoint_dir)
|
wandb_logger.log_policy(checkpoint_dir)
|
||||||
|
|
||||||
if cfg.env and is_eval_step and (not accelerator or accelerator.is_main_process):
|
if cfg.env and is_eval_step:
|
||||||
step_id = get_step_identifier(step, cfg.steps)
|
step_id = get_step_identifier(step, cfg.steps)
|
||||||
logging.info(f"Eval policy at step {step}")
|
logging.info(f"Eval policy at step {step}")
|
||||||
with (
|
with (
|
||||||
@@ -323,7 +254,7 @@ def train(cfg: TrainPipelineConfig):
|
|||||||
):
|
):
|
||||||
eval_info = eval_policy(
|
eval_info = eval_policy(
|
||||||
eval_env,
|
eval_env,
|
||||||
accelerator.unwrap_model(policy) if accelerator else policy,
|
policy,
|
||||||
cfg.eval.n_episodes,
|
cfg.eval.n_episodes,
|
||||||
videos_dir=cfg.output_dir / "eval" / f"videos_step_{step_id}",
|
videos_dir=cfg.output_dir / "eval" / f"videos_step_{step_id}",
|
||||||
max_episodes_rendered=4,
|
max_episodes_rendered=4,
|
||||||
|
|||||||
@@ -60,39 +60,11 @@ def load_training_step(save_dir: Path) -> int:
|
|||||||
|
|
||||||
|
|
||||||
def update_last_checkpoint(checkpoint_dir: Path) -> Path:
|
def update_last_checkpoint(checkpoint_dir: Path) -> Path:
|
||||||
import fcntl
|
|
||||||
import tempfile
|
|
||||||
import os
|
|
||||||
|
|
||||||
last_checkpoint_dir = checkpoint_dir.parent / LAST_CHECKPOINT_LINK
|
last_checkpoint_dir = checkpoint_dir.parent / LAST_CHECKPOINT_LINK
|
||||||
|
if last_checkpoint_dir.is_symlink():
|
||||||
|
last_checkpoint_dir.unlink()
|
||||||
relative_target = checkpoint_dir.relative_to(checkpoint_dir.parent)
|
relative_target = checkpoint_dir.relative_to(checkpoint_dir.parent)
|
||||||
|
last_checkpoint_dir.symlink_to(relative_target)
|
||||||
# Use file locking to prevent race conditions in multi-GPU training
|
|
||||||
lock_file = checkpoint_dir.parent / ".symlink_lock"
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(lock_file, 'w') as f:
|
|
||||||
# Get exclusive lock
|
|
||||||
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
|
||||||
|
|
||||||
# Update symlink atomically
|
|
||||||
if last_checkpoint_dir.exists() or last_checkpoint_dir.is_symlink():
|
|
||||||
last_checkpoint_dir.unlink()
|
|
||||||
last_checkpoint_dir.symlink_to(relative_target)
|
|
||||||
|
|
||||||
except (OSError, FileExistsError) as e:
|
|
||||||
# Handle race conditions gracefully - another process may have already updated
|
|
||||||
if not last_checkpoint_dir.exists():
|
|
||||||
try:
|
|
||||||
last_checkpoint_dir.symlink_to(relative_target)
|
|
||||||
except FileExistsError:
|
|
||||||
pass # Another process created it, that's fine
|
|
||||||
finally:
|
|
||||||
# Clean up lock file
|
|
||||||
try:
|
|
||||||
lock_file.unlink()
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def save_checkpoint(
|
def save_checkpoint(
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
echo "=== Local 1-GPU Accelerate Training Test with SmolVLA ==="
|
|
||||||
echo "Environment: multi"
|
|
||||||
echo "GPU: 1"
|
|
||||||
echo "Steps: 50 (quick local test)"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Activate conda environment
|
|
||||||
source /fsx/dana_aubakirova/miniconda3/etc/profile.d/conda.sh
|
|
||||||
conda activate multi
|
|
||||||
|
|
||||||
# Set CUDA environment for 1 GPU
|
|
||||||
export CUDA_VISIBLE_DEVICES=0
|
|
||||||
export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128,expandable_segments:True
|
|
||||||
export TORCH_DISTRIBUTED_DEBUG=OFF
|
|
||||||
export CUDA_LAUNCH_BLOCKING=0
|
|
||||||
export TRANSFORMERS_NO_ADVISORY_WARNINGS=1
|
|
||||||
|
|
||||||
# Change to working directory
|
|
||||||
cd /fsx/dana_aubakirova/vla/pr/lerobot
|
|
||||||
|
|
||||||
# Set output directory with timestamp
|
|
||||||
export OUTPUT_DIR="outputs/test_accelerate_1gpu_local_$(date +%Y%m%d_%H%M%S)"
|
|
||||||
|
|
||||||
echo "Output directory: $OUTPUT_DIR"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Test accelerate training with 1 GPU
|
|
||||||
accelerate launch --config_file accelerate_configs/1gpu_config.yaml -m lerobot.scripts.train \
|
|
||||||
--policy.path=lerobot/smolvla_base \
|
|
||||||
--policy.push_to_hub=false \
|
|
||||||
--dataset.repo_id=lerobot/svla_so100_sorting \
|
|
||||||
--dataset.video_backend=pyav \
|
|
||||||
--steps=50 \
|
|
||||||
--save_freq=25 \
|
|
||||||
--log_freq=5 \
|
|
||||||
--batch_size=1 \
|
|
||||||
--num_workers=0 \
|
|
||||||
--output_dir=$OUTPUT_DIR \
|
|
||||||
--wandb.enable=false
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Training completed! ==="
|
|
||||||
echo "Check outputs in: $OUTPUT_DIR"
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#SBATCH --job-name=test_accelerate
|
|
||||||
#SBATCH --nodes=1
|
|
||||||
#SBATCH --ntasks-per-node=1
|
|
||||||
#SBATCH --cpus-per-task=16
|
|
||||||
#SBATCH --gres=gpu:2
|
|
||||||
#SBATCH --time=1:00:00
|
|
||||||
#SBATCH --partition=hopper-prod
|
|
||||||
#SBATCH --output=/fsx/dana_aubakirova/vla/logs/test_accelerate_%j.out
|
|
||||||
#SBATCH --error=/fsx/dana_aubakirova/vla/logs/test_accelerate_%j.err
|
|
||||||
|
|
||||||
# Create logs directory if it doesn't exist
|
|
||||||
mkdir -p /fsx/dana_aubakirova/vla/pr/lerobot/logs
|
|
||||||
|
|
||||||
# Activate conda environment
|
|
||||||
source /fsx/dana_aubakirova/miniconda3/etc/profile.d/conda.sh
|
|
||||||
conda activate multi
|
|
||||||
|
|
||||||
# 2-GPU Test CUDA environment
|
|
||||||
export CUDA_VISIBLE_DEVICES=0,1
|
|
||||||
export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128,expandable_segments:True
|
|
||||||
export TORCH_DISTRIBUTED_DEBUG=OFF
|
|
||||||
export NCCL_DEBUG=INFO
|
|
||||||
export CUDA_LAUNCH_BLOCKING=0
|
|
||||||
export ACCELERATE_USE_FSDP=false
|
|
||||||
export ACCELERATE_USE_DEEPSPEED=false
|
|
||||||
export HF_ACCELERATE_DEVICE_MAP=false
|
|
||||||
export TRANSFORMERS_NO_ADVISORY_WARNINGS=1
|
|
||||||
export SAFETENSORS_FAST_GPU=1
|
|
||||||
export HF_HUB_ENABLE_HF_TRANSFER=1
|
|
||||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
|
||||||
export ACCELERATE_TORCH_DEVICE_MAP_AUTO=false
|
|
||||||
|
|
||||||
# Change to working directory
|
|
||||||
cd /fsx/dana_aubakirova/vla/pr/lerobot
|
|
||||||
|
|
||||||
echo "=== Testing Accelerate Multi-GPU Training with SmolVLA ==="
|
|
||||||
echo "Dataset: lerobot/svla_so100_sorting"
|
|
||||||
echo "GPUs: 2"
|
|
||||||
echo "Steps: 100 (for quick test)"
|
|
||||||
echo "Job ID: $SLURM_JOB_ID"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Set output directory with job ID
|
|
||||||
export OUTPUT_DIR="outputs/test_accelerate_2gpu_job_${SLURM_JOB_ID}"
|
|
||||||
|
|
||||||
echo "Output directory: $OUTPUT_DIR"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Test accelerate training
|
|
||||||
accelerate launch --config_file accelerate_configs/2gpu_config_safe.yaml -m lerobot.scripts.train \
|
|
||||||
--policy.type=smolvla \
|
|
||||||
--policy.push_to_hub=false \
|
|
||||||
--dataset.repo_id=lerobot/svla_so100_sorting \
|
|
||||||
--dataset.video_backend=pyav \
|
|
||||||
--steps=100 \
|
|
||||||
--save_freq=50 \
|
|
||||||
--log_freq=5 \
|
|
||||||
--batch_size=2 \
|
|
||||||
--num_workers=0 \
|
|
||||||
--output_dir=$OUTPUT_DIR \
|
|
||||||
--wandb.enable=false
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Training completed! ==="
|
|
||||||
echo "Check logs and outputs in: $OUTPUT_DIR"
|
|
||||||
echo "Job ID: $SLURM_JOB_ID"
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
echo "=== Direct 1-GPU Training Test with SmolVLA (no accelerate) ==="
|
|
||||||
echo "Environment: multi"
|
|
||||||
echo "GPU: 1"
|
|
||||||
echo "Steps: 50 (quick local test)"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Activate conda environment
|
|
||||||
source /fsx/dana_aubakirova/miniconda3/etc/profile.d/conda.sh
|
|
||||||
conda activate multi
|
|
||||||
|
|
||||||
# Set CUDA environment for 1 GPU
|
|
||||||
export CUDA_VISIBLE_DEVICES=0
|
|
||||||
export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128,expandable_segments:True
|
|
||||||
export TORCH_DISTRIBUTED_DEBUG=OFF
|
|
||||||
export CUDA_LAUNCH_BLOCKING=0
|
|
||||||
export TRANSFORMERS_NO_ADVISORY_WARNINGS=1
|
|
||||||
|
|
||||||
# Change to working directory
|
|
||||||
cd /fsx/dana_aubakirova/vla/pr/lerobot
|
|
||||||
|
|
||||||
# Set output directory with timestamp
|
|
||||||
export OUTPUT_DIR="outputs/test_direct_1gpu_local_$(date +%Y%m%d_%H%M%S)"
|
|
||||||
|
|
||||||
echo "Output directory: $OUTPUT_DIR"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# Test direct training with 1 GPU (no accelerate)
|
|
||||||
python -m lerobot.scripts.train \
|
|
||||||
--policy.path=lerobot/smolvla_base \
|
|
||||||
--policy.push_to_hub=false \
|
|
||||||
--dataset.repo_id=lerobot/svla_so100_sorting \
|
|
||||||
--dataset.video_backend=pyav \
|
|
||||||
--steps=50 \
|
|
||||||
--save_freq=25 \
|
|
||||||
--log_freq=5 \
|
|
||||||
--batch_size=1 \
|
|
||||||
--num_workers=0 \
|
|
||||||
--output_dir=$OUTPUT_DIR \
|
|
||||||
--wandb.enable=false
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Training completed! ==="
|
|
||||||
echo "Check outputs in: $OUTPUT_DIR"
|
|
||||||
182
visualizer/index.html
Normal file
182
visualizer/index.html
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>3D Hand Joint Visualizer</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-family: Arial, sans-serif;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
height: 100vh;
|
||||||
|
}
|
||||||
|
|
||||||
|
.controls {
|
||||||
|
padding: 15px;
|
||||||
|
background-color: #f5f5f5;
|
||||||
|
z-index: 100;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status {
|
||||||
|
padding: 10px;
|
||||||
|
border-radius: 5px;
|
||||||
|
margin: 10px 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.connected {
|
||||||
|
background-color: #d4edda;
|
||||||
|
color: #155724;
|
||||||
|
}
|
||||||
|
|
||||||
|
.disconnected {
|
||||||
|
background-color: #f8d7da;
|
||||||
|
color: #721c24;
|
||||||
|
}
|
||||||
|
|
||||||
|
button {
|
||||||
|
padding: 8px 16px;
|
||||||
|
background-color: #4CAF50;
|
||||||
|
color: white;
|
||||||
|
border: none;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
margin-right: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:hover {
|
||||||
|
background-color: #45a049;
|
||||||
|
}
|
||||||
|
|
||||||
|
button:disabled {
|
||||||
|
background-color: #cccccc;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.container {
|
||||||
|
display: flex;
|
||||||
|
flex: 1;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
#canvas-container {
|
||||||
|
flex: 3;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sidebar {
|
||||||
|
flex: 1;
|
||||||
|
padding: 15px;
|
||||||
|
background-color: #f8f9fa;
|
||||||
|
overflow-y: auto;
|
||||||
|
max-width: 300px;
|
||||||
|
border-left: 1px solid #ddd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.joint-info {
|
||||||
|
margin-bottom: 10px;
|
||||||
|
padding: 8px;
|
||||||
|
border: 1px solid #ddd;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.joint-name {
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.joint-value {
|
||||||
|
font-family: monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bar-container {
|
||||||
|
width: 100%;
|
||||||
|
background-color: #e0e0e0;
|
||||||
|
height: 10px;
|
||||||
|
border-radius: 5px;
|
||||||
|
overflow: hidden;
|
||||||
|
margin-top: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bar {
|
||||||
|
height: 100%;
|
||||||
|
background-color: #4CAF50;
|
||||||
|
width: 0%;
|
||||||
|
transition: width 0.2s ease-in-out;
|
||||||
|
}
|
||||||
|
|
||||||
|
.log-container {
|
||||||
|
margin-top: 20px;
|
||||||
|
border: 1px solid #ddd;
|
||||||
|
border-radius: 5px;
|
||||||
|
padding: 10px;
|
||||||
|
height: 150px;
|
||||||
|
overflow-y: auto;
|
||||||
|
font-family: monospace;
|
||||||
|
background-color: #f8f9fa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.view-controls {
|
||||||
|
position: absolute;
|
||||||
|
bottom: 10px;
|
||||||
|
left: 10px;
|
||||||
|
z-index: 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
.view-button {
|
||||||
|
background-color: rgba(0, 0, 0, 0.5);
|
||||||
|
color: white;
|
||||||
|
border: none;
|
||||||
|
padding: 5px 10px;
|
||||||
|
margin-right: 5px;
|
||||||
|
border-radius: 3px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="controls">
|
||||||
|
<button id="connectButton">Connect to Device</button>
|
||||||
|
<button id="disconnectButton" disabled>Disconnect</button>
|
||||||
|
<select id="baudRate">
|
||||||
|
<option value="9600">9600</option>
|
||||||
|
<option value="19200">19200</option>
|
||||||
|
<option value="38400">38400</option>
|
||||||
|
<option value="57600">57600</option>
|
||||||
|
<option value="115200" selected>115200</option>
|
||||||
|
</select>
|
||||||
|
<span id="statusIndicator" class="status disconnected">Status: Disconnected</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="container">
|
||||||
|
<div id="canvas-container">
|
||||||
|
<!-- 3D canvas will be inserted here -->
|
||||||
|
<div class="view-controls">
|
||||||
|
<button class="view-button" id="frontView">Front</button>
|
||||||
|
<button class="view-button" id="sideView">Side</button>
|
||||||
|
<button class="view-button" id="topView">Top</button>
|
||||||
|
<button class="view-button" id="resetView">Reset</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="sidebar">
|
||||||
|
<h3>Joint Values</h3>
|
||||||
|
<div id="jointsContainer">
|
||||||
|
<!-- Joint info will be added here -->
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="log-container" id="logContainer">
|
||||||
|
<!-- Log messages will be added here -->
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Import Three.js -->
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/three@0.132.2/build/three.min.js"></script>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/three@0.132.2/examples/js/controls/OrbitControls.js"></script>
|
||||||
|
|
||||||
|
<script src="script.js"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
669
visualizer/script.js
Normal file
669
visualizer/script.js
Normal file
@@ -0,0 +1,669 @@
|
|||||||
|
// === Hand Visualizer with Pre-Connect Sliders + Per-Joint Angle Limits ===
|
||||||
|
// Assumes your HTML already has elements with the following IDs:
|
||||||
|
// connectButton, disconnectButton, baudRate, statusIndicator, jointsContainer, logContainer,
|
||||||
|
// canvas-container, frontView, sideView, topView, resetView
|
||||||
|
// Requires Three.js + OrbitControls loaded on the page.
|
||||||
|
|
||||||
|
// -------------------- Config --------------------
|
||||||
|
const MAX_JOINTS = 16;
|
||||||
|
const RAW_MIN = 0, RAW_MAX = 4096;
|
||||||
|
const RAW_CENTER = (RAW_MIN + RAW_MAX) / 2;
|
||||||
|
const DEG = Math.PI / 180;
|
||||||
|
const UI_DEG_MIN = -90, UI_DEG_MAX = 90; // UI sliders for angle limits
|
||||||
|
|
||||||
|
// -------------------- State --------------------
|
||||||
|
let port;
|
||||||
|
let reader;
|
||||||
|
let keepReading = false;
|
||||||
|
let isConnected = false;
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let inputBuffer = '';
|
||||||
|
|
||||||
|
let jointValues = new Array(MAX_JOINTS).fill(RAW_CENTER);
|
||||||
|
|
||||||
|
// Auto-calibration: track observed min/max per joint
|
||||||
|
let observedMin = new Array(MAX_JOINTS).fill(Infinity);
|
||||||
|
let observedMax = new Array(MAX_JOINTS).fill(-Infinity);
|
||||||
|
let calibrationEnabled = true;
|
||||||
|
|
||||||
|
// Three.js
|
||||||
|
let scene, camera, renderer, controls;
|
||||||
|
let hand = { palm: null, fingers: [] };
|
||||||
|
|
||||||
|
// DOM
|
||||||
|
const connectButton = document.getElementById('connectButton');
|
||||||
|
const disconnectButton = document.getElementById('disconnectButton');
|
||||||
|
const baudRateSelect = document.getElementById('baudRate');
|
||||||
|
const statusIndicator = document.getElementById('statusIndicator');
|
||||||
|
const jointsContainer = document.getElementById('jointsContainer');
|
||||||
|
const logContainer = document.getElementById('logContainer');
|
||||||
|
const canvasContainer = document.getElementById('canvas-container');
|
||||||
|
const frontViewBtn = document.getElementById('frontView');
|
||||||
|
const sideViewBtn = document.getElementById('sideView');
|
||||||
|
const topViewBtn = document.getElementById('topView');
|
||||||
|
const resetViewBtn = document.getElementById('resetView');
|
||||||
|
|
||||||
|
// Helpers
|
||||||
|
const clamp = (x, a, b) => Math.max(a, Math.min(b, x));
|
||||||
|
const invLerp = (a, b, x) => clamp((x - a) / (b - a), 0, 1);
|
||||||
|
|
||||||
|
// -------------------- Joint Map with per-joint angle limits --------------------
|
||||||
|
const fingerJointMap = [
|
||||||
|
// Thumb (4)
|
||||||
|
{ finger:0, joint:0, type:'CMC_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:0, joint:1, type:'CMC_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:0, joint:2, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
{ finger:0, joint:3, type:'IP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
|
||||||
|
// Index (3)
|
||||||
|
{ finger:1, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:1, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:1, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
|
||||||
|
// Middle (3)
|
||||||
|
{ finger:2, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:2, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:2, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:true }, // +45° only
|
||||||
|
|
||||||
|
// Ring (3)
|
||||||
|
{ finger:3, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:true },
|
||||||
|
{ finger:3, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:3, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false }, // +45° only
|
||||||
|
|
||||||
|
// Pinky (3)
|
||||||
|
{ finger:4, joint:0, type:'MCP_ABDUCTION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:4, joint:1, type:'MCP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false },
|
||||||
|
{ finger:4, joint:2, type:'PIP_FLEXION', min:RAW_MIN, max:RAW_MAX, inverted:false } // +45° only
|
||||||
|
];
|
||||||
|
|
||||||
|
// Assign angle limits (radians) per joint (default ±45°, exceptions: +45° only)
|
||||||
|
for (const j of fingerJointMap) {
|
||||||
|
const isThumb = j.finger === 0;
|
||||||
|
const isPIP = j.type === 'PIP_FLEXION';
|
||||||
|
let minA = -45 * DEG, maxA = +45 * DEG;
|
||||||
|
if ((isThumb && (j.type === 'MCP_FLEXION' || j.type === 'IP_FLEXION')) || (!isThumb && isPIP)) {
|
||||||
|
minA = 0;
|
||||||
|
maxA = +45 * DEG;
|
||||||
|
}
|
||||||
|
j.angleMin = minA;
|
||||||
|
j.angleMax = maxA;
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- UI: Joint Panel --------------------
|
||||||
|
const uiRefs = []; // per joint: { valueLabel, bar, barWrap, slider, invertChk, minDeg, maxDeg }
|
||||||
|
|
||||||
|
function initializeJointElements() {
|
||||||
|
jointsContainer.innerHTML = '';
|
||||||
|
uiRefs.length = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < MAX_JOINTS; i++) {
|
||||||
|
const wrap = document.createElement('div');
|
||||||
|
wrap.className = 'joint-info';
|
||||||
|
|
||||||
|
const fingerIndex = i < 4 ? 0 : Math.floor((i - 4) / 3) + 1;
|
||||||
|
const jointInfo = fingerJointMap[i];
|
||||||
|
const jointType = jointInfo?.type || 'Unknown';
|
||||||
|
const fingerName = ['Thumb', 'Index', 'Middle', 'Ring', 'Pinky'][fingerIndex];
|
||||||
|
|
||||||
|
// Header
|
||||||
|
const nameEl = document.createElement('div');
|
||||||
|
nameEl.className = 'joint-name';
|
||||||
|
nameEl.textContent = `${fingerName} – ${jointType}`;
|
||||||
|
|
||||||
|
// Value + bar
|
||||||
|
const valueEl = document.createElement('div');
|
||||||
|
valueEl.className = 'joint-value';
|
||||||
|
valueEl.textContent = `Value: ${jointValues[i]}`;
|
||||||
|
|
||||||
|
const barWrap = document.createElement('div');
|
||||||
|
barWrap.className = 'bar-container';
|
||||||
|
const barEl = document.createElement('div');
|
||||||
|
barEl.className = 'bar';
|
||||||
|
barWrap.appendChild(barEl);
|
||||||
|
|
||||||
|
// Slider for pre-connect manual control
|
||||||
|
const slider = document.createElement('input');
|
||||||
|
slider.type = 'range';
|
||||||
|
slider.min = String(RAW_MIN);
|
||||||
|
slider.max = String(RAW_MAX);
|
||||||
|
slider.value = String(jointValues[i]);
|
||||||
|
slider.step = '1';
|
||||||
|
slider.className = 'joint-slider';
|
||||||
|
|
||||||
|
slider.addEventListener('input', () => {
|
||||||
|
if (isConnected) return; // ignore while connected
|
||||||
|
let v = parseInt(slider.value, 10);
|
||||||
|
if (jointInfo?.inverted) v = (jointInfo.min + jointInfo.max) - v;
|
||||||
|
jointValues[i] = clamp(jointInfo ? v : 0, RAW_MIN, RAW_MAX);
|
||||||
|
updateJointDisplay(i, jointValues[i]);
|
||||||
|
updateHandModel();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Invert checkbox
|
||||||
|
const invertLbl = document.createElement('label');
|
||||||
|
invertLbl.className = 'invert-toggle';
|
||||||
|
const invertChk = document.createElement('input');
|
||||||
|
invertChk.type = 'checkbox';
|
||||||
|
invertChk.checked = !!jointInfo?.inverted;
|
||||||
|
invertChk.addEventListener('change', () => {
|
||||||
|
if (jointInfo) jointInfo.inverted = invertChk.checked;
|
||||||
|
addLogMessage(`${fingerName} ${jointType} inversion ${invertChk.checked ? 'enabled' : 'disabled'}`);
|
||||||
|
});
|
||||||
|
invertLbl.appendChild(invertChk);
|
||||||
|
invertLbl.appendChild(document.createTextNode('Invert Values'));
|
||||||
|
|
||||||
|
// Angle limits (deg) controls
|
||||||
|
const limitsRow = document.createElement('div');
|
||||||
|
limitsRow.className = 'limits-row';
|
||||||
|
|
||||||
|
const minDeg = document.createElement('input');
|
||||||
|
minDeg.type = 'number';
|
||||||
|
minDeg.min = String(UI_DEG_MIN);
|
||||||
|
minDeg.max = String(UI_DEG_MAX);
|
||||||
|
minDeg.step = '1';
|
||||||
|
minDeg.value = String(Math.round((jointInfo.angleMin || 0) / DEG));
|
||||||
|
minDeg.className = 'limit-num';
|
||||||
|
|
||||||
|
const maxDeg = document.createElement('input');
|
||||||
|
maxDeg.type = 'number';
|
||||||
|
maxDeg.min = String(UI_DEG_MIN);
|
||||||
|
maxDeg.max = String(UI_DEG_MAX);
|
||||||
|
maxDeg.step = '1';
|
||||||
|
maxDeg.value = String(Math.round((jointInfo.angleMax || 0) / DEG));
|
||||||
|
maxDeg.className = 'limit-num';
|
||||||
|
|
||||||
|
const minLbl = document.createElement('span'); minLbl.textContent = 'min°';
|
||||||
|
const maxLbl = document.createElement('span'); maxLbl.textContent = 'max°';
|
||||||
|
minLbl.className = 'limit-label'; maxLbl.className = 'limit-label';
|
||||||
|
|
||||||
|
function syncLimits() {
|
||||||
|
let mn = parseFloat(minDeg.value);
|
||||||
|
let mx = parseFloat(maxDeg.value);
|
||||||
|
if (isNaN(mn)) mn = -45;
|
||||||
|
if (isNaN(mx)) mx = +45;
|
||||||
|
if (mn > mx) [mn, mx] = [mx, mn];
|
||||||
|
jointInfo.angleMin = clamp(mn, UI_DEG_MIN, UI_DEG_MAX) * DEG;
|
||||||
|
jointInfo.angleMax = clamp(mx, UI_DEG_MIN, UI_DEG_MAX) * DEG;
|
||||||
|
minDeg.value = String(Math.round(jointInfo.angleMin / DEG));
|
||||||
|
maxDeg.value = String(Math.round(jointInfo.angleMax / DEG));
|
||||||
|
updateHandModel();
|
||||||
|
}
|
||||||
|
minDeg.addEventListener('change', syncLimits);
|
||||||
|
maxDeg.addEventListener('change', syncLimits);
|
||||||
|
|
||||||
|
limitsRow.appendChild(minLbl);
|
||||||
|
limitsRow.appendChild(minDeg);
|
||||||
|
limitsRow.appendChild(maxLbl);
|
||||||
|
limitsRow.appendChild(maxDeg);
|
||||||
|
|
||||||
|
// Calibration controls
|
||||||
|
const calibRow = document.createElement('div');
|
||||||
|
calibRow.className = 'calib-row';
|
||||||
|
|
||||||
|
const resetCalibBtn = document.createElement('button');
|
||||||
|
resetCalibBtn.textContent = 'Reset Calib';
|
||||||
|
resetCalibBtn.className = 'calib-btn';
|
||||||
|
resetCalibBtn.addEventListener('click', () => {
|
||||||
|
observedMin[i] = Infinity;
|
||||||
|
observedMax[i] = -Infinity;
|
||||||
|
addLogMessage(`Reset calibration for ${fingerName} ${jointType}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
const calibStatus = document.createElement('span');
|
||||||
|
calibStatus.className = 'calib-status';
|
||||||
|
calibStatus.textContent = `Range: --`;
|
||||||
|
|
||||||
|
calibRow.appendChild(resetCalibBtn);
|
||||||
|
calibRow.appendChild(calibStatus);
|
||||||
|
|
||||||
|
// Compose
|
||||||
|
wrap.appendChild(nameEl);
|
||||||
|
wrap.appendChild(valueEl);
|
||||||
|
wrap.appendChild(barWrap);
|
||||||
|
wrap.appendChild(slider);
|
||||||
|
wrap.appendChild(invertLbl);
|
||||||
|
wrap.appendChild(limitsRow);
|
||||||
|
wrap.appendChild(calibRow);
|
||||||
|
|
||||||
|
jointsContainer.appendChild(wrap);
|
||||||
|
|
||||||
|
uiRefs[i] = { valueLabel: valueEl, bar: barEl, barWrap, slider, invertChk, minDeg, maxDeg, nameEl, calibStatus };
|
||||||
|
}
|
||||||
|
|
||||||
|
setConnectedUI(false); // initial state: sliders active
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toggle UI between pre-connect SLIDERS vs post-connect BARS
|
||||||
|
function setConnectedUI(connected) {
|
||||||
|
isConnected = connected;
|
||||||
|
for (let i = 0; i < uiRefs.length; i++) {
|
||||||
|
const ui = uiRefs[i];
|
||||||
|
if (!ui) continue;
|
||||||
|
// Show bars when connected; sliders disabled/hidden
|
||||||
|
ui.barWrap.style.display = connected ? '' : 'none';
|
||||||
|
ui.slider.disabled = connected;
|
||||||
|
ui.slider.style.display = connected ? 'none' : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset calibration when connecting
|
||||||
|
if (connected) {
|
||||||
|
observedMin.fill(Infinity);
|
||||||
|
observedMax.fill(-Infinity);
|
||||||
|
addLogMessage('Calibration reset - move joints through full range for best results');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update joint display (value text + bar color/width + slider position if needed)
|
||||||
|
function updateJointDisplay(jointIndex, value) {
|
||||||
|
const ui = uiRefs[jointIndex];
|
||||||
|
const info = fingerJointMap[jointIndex];
|
||||||
|
if (!ui || !info) return;
|
||||||
|
|
||||||
|
ui.valueLabel.textContent = `Value: ${value}`;
|
||||||
|
|
||||||
|
// bar
|
||||||
|
const min = info.min, max = info.max;
|
||||||
|
const pct = clamp((value - min) / (max - min), 0, 1) * 100;
|
||||||
|
ui.bar.style.width = `${pct}%`;
|
||||||
|
const hue = Math.floor(pct * 1.2); // 0..120
|
||||||
|
ui.bar.style.backgroundColor = `hsl(${hue}, 80%, 50%)`;
|
||||||
|
|
||||||
|
// slider (only meaningful when not connected; keep in sync anyway)
|
||||||
|
const rawForSlider = info.inverted ? (info.min + info.max) - value : value;
|
||||||
|
if (!isConnected) ui.slider.value = String(clamp(Math.round(rawForSlider), RAW_MIN, RAW_MAX));
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Serial I/O --------------------
|
||||||
|
async function readSerialData() {
|
||||||
|
while (port?.readable && keepReading) {
|
||||||
|
reader = port.readable.getReader();
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const { value, done } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
if (value) processData(decoder.decode(value));
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error reading:', err);
|
||||||
|
addLogMessage(`Error: ${err.message}`);
|
||||||
|
break;
|
||||||
|
} finally {
|
||||||
|
reader.releaseLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function processData(chunk) {
|
||||||
|
inputBuffer += chunk;
|
||||||
|
let idx;
|
||||||
|
while ((idx = inputBuffer.indexOf('\n')) !== -1) {
|
||||||
|
const line = inputBuffer.slice(0, idx).trim();
|
||||||
|
inputBuffer = inputBuffer.slice(idx + 1);
|
||||||
|
|
||||||
|
const vals = line.split(/\s+/).map(v => parseInt(v, 10));
|
||||||
|
if (vals.length === MAX_JOINTS && vals.every(v => Number.isFinite(v))) {
|
||||||
|
for (let i = 0; i < MAX_JOINTS; i++) {
|
||||||
|
const info = fingerJointMap[i];
|
||||||
|
if (!info) continue;
|
||||||
|
|
||||||
|
let rawValue = vals[i];
|
||||||
|
|
||||||
|
// Update calibration tracking
|
||||||
|
if (calibrationEnabled) {
|
||||||
|
observedMin[i] = Math.min(observedMin[i], rawValue);
|
||||||
|
observedMax[i] = Math.max(observedMax[i], rawValue);
|
||||||
|
|
||||||
|
// Update calibration display
|
||||||
|
const ui = uiRefs[i];
|
||||||
|
if (ui && ui.calibStatus) {
|
||||||
|
if (observedMin[i] !== Infinity && observedMax[i] !== -Infinity) {
|
||||||
|
ui.calibStatus.textContent = `Range: ${observedMin[i]}-${observedMax[i]}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remap observed range to target range
|
||||||
|
if (observedMin[i] !== Infinity && observedMax[i] !== -Infinity && observedMax[i] > observedMin[i]) {
|
||||||
|
const observedRange = observedMax[i] - observedMin[i];
|
||||||
|
const targetRange = info.max - info.min;
|
||||||
|
const normalizedValue = (rawValue - observedMin[i]) / observedRange;
|
||||||
|
rawValue = info.min + (normalizedValue * targetRange);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let v = clamp(rawValue, info.min, info.max);
|
||||||
|
if (info.inverted) v = (info.min + info.max) - v;
|
||||||
|
jointValues[i] = v;
|
||||||
|
updateJointDisplay(i, v);
|
||||||
|
}
|
||||||
|
updateHandModel();
|
||||||
|
} else {
|
||||||
|
addLogMessage(`Received: ${line}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function connectToDevice() {
|
||||||
|
try {
|
||||||
|
port = await navigator.serial.requestPort();
|
||||||
|
const baudRate = parseInt(baudRateSelect.value, 10) || 115200;
|
||||||
|
await port.open({ baudRate });
|
||||||
|
|
||||||
|
keepReading = true;
|
||||||
|
setConnectedUI(true);
|
||||||
|
|
||||||
|
statusIndicator.textContent = 'Status: Connected';
|
||||||
|
statusIndicator.className = 'status connected';
|
||||||
|
connectButton.disabled = true;
|
||||||
|
disconnectButton.disabled = false;
|
||||||
|
baudRateSelect.disabled = true;
|
||||||
|
|
||||||
|
addLogMessage(`Connected at ${baudRate} baud`);
|
||||||
|
readSerialData();
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Connect error:', e);
|
||||||
|
addLogMessage(`Connection error: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function disconnectFromDevice() {
|
||||||
|
try {
|
||||||
|
keepReading = false;
|
||||||
|
if (reader) {
|
||||||
|
try { reader.cancel(); } catch {}
|
||||||
|
}
|
||||||
|
if (port) {
|
||||||
|
await port.close();
|
||||||
|
port = null;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Disconnect error:', e);
|
||||||
|
addLogMessage(`Disconnection error: ${e.message}`);
|
||||||
|
} finally {
|
||||||
|
setConnectedUI(false);
|
||||||
|
statusIndicator.textContent = 'Status: Disconnected';
|
||||||
|
statusIndicator.className = 'status disconnected';
|
||||||
|
connectButton.disabled = false;
|
||||||
|
disconnectButton.disabled = true;
|
||||||
|
baudRateSelect.disabled = false;
|
||||||
|
addLogMessage('Disconnected');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Three.js Scene --------------------
|
||||||
|
function initThreeJS() {
|
||||||
|
scene = new THREE.Scene();
|
||||||
|
scene.background = new THREE.Color(0xf0f0f0);
|
||||||
|
|
||||||
|
camera = new THREE.PerspectiveCamera(
|
||||||
|
75,
|
||||||
|
canvasContainer.clientWidth / canvasContainer.clientHeight,
|
||||||
|
0.1, 1000
|
||||||
|
);
|
||||||
|
camera.position.set(0, 15, 15);
|
||||||
|
camera.lookAt(0, 0, 0);
|
||||||
|
|
||||||
|
renderer = new THREE.WebGLRenderer({ antialias: true });
|
||||||
|
renderer.setSize(canvasContainer.clientWidth, canvasContainer.clientHeight);
|
||||||
|
renderer.setPixelRatio(window.devicePixelRatio);
|
||||||
|
canvasContainer.appendChild(renderer.domElement);
|
||||||
|
|
||||||
|
controls = new THREE.OrbitControls(camera, renderer.domElement);
|
||||||
|
controls.enableDamping = true;
|
||||||
|
controls.dampingFactor = 0.25;
|
||||||
|
|
||||||
|
const ambientLight = new THREE.AmbientLight(0x404040);
|
||||||
|
scene.add(ambientLight);
|
||||||
|
const dir1 = new THREE.DirectionalLight(0xffffff, 0.5);
|
||||||
|
dir1.position.set(1, 1, 1);
|
||||||
|
scene.add(dir1);
|
||||||
|
const dir2 = new THREE.DirectionalLight(0xffffff, 0.3);
|
||||||
|
dir2.position.set(-1, 1, -1);
|
||||||
|
scene.add(dir2);
|
||||||
|
|
||||||
|
const gridHelper = new THREE.GridHelper(20, 20);
|
||||||
|
scene.add(gridHelper);
|
||||||
|
|
||||||
|
createHandModel();
|
||||||
|
window.addEventListener('resize', onWindowResize);
|
||||||
|
animate();
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHandModel() {
|
||||||
|
const palmMaterial = new THREE.MeshPhongMaterial({ color: 0xf5c396 });
|
||||||
|
const fingerMaterial = new THREE.MeshPhongMaterial({ color: 0xf5c396 });
|
||||||
|
const jointMaterial = new THREE.MeshPhongMaterial({ color: 0xe3a977 });
|
||||||
|
|
||||||
|
const palmGeometry = new THREE.BoxGeometry(7, 1, 8);
|
||||||
|
hand.palm = new THREE.Mesh(palmGeometry, palmMaterial);
|
||||||
|
hand.palm.position.set(0, 0, 0);
|
||||||
|
hand.palm.rotation.x = Math.PI / 2; // hand vertical, palm facing forward
|
||||||
|
scene.add(hand.palm);
|
||||||
|
|
||||||
|
const fingerWidth = 1, fingerHeight = 0.8;
|
||||||
|
const fingerSegmentLengths = [3, 2, 1.5];
|
||||||
|
const thumbSegmentLengths = [2, 2, 1.5];
|
||||||
|
|
||||||
|
const fingerBasePositions = [
|
||||||
|
[ 3, 0, -2], // Thumb
|
||||||
|
[ 1.5,-0.5,-4], // Index
|
||||||
|
[ 0, -0.5,-4], // Middle
|
||||||
|
[-1.5,-0.5,-4], // Ring
|
||||||
|
[-3, -0.5,-4], // Pinky
|
||||||
|
];
|
||||||
|
const fingerBaseRot = [
|
||||||
|
{ x:0, y:-Math.PI/3, z: Math.PI/3 }, // Thumb
|
||||||
|
{ x:0, y:-Math.PI/48, z: 0 },
|
||||||
|
{ x:0, y: Math.PI/48, z: 0 },
|
||||||
|
{ x:0, y: Math.PI/32, z: 0 },
|
||||||
|
{ x:0, y: Math.PI/24, z: 0 }
|
||||||
|
];
|
||||||
|
|
||||||
|
for (let fIdx = 0; fIdx < 5; fIdx++) {
|
||||||
|
const finger = { name:['Thumb','Index','Middle','Ring','Pinky'][fIdx], segments:[], joints:[] };
|
||||||
|
const isThumb = fIdx === 0;
|
||||||
|
const segLens = isThumb ? thumbSegmentLengths : fingerSegmentLengths;
|
||||||
|
|
||||||
|
finger.group = new THREE.Group();
|
||||||
|
finger.group.position.set(...fingerBasePositions[fIdx]);
|
||||||
|
finger.group.rotation.x = fingerBaseRot[fIdx].x;
|
||||||
|
finger.group.rotation.y = fingerBaseRot[fIdx].y;
|
||||||
|
finger.group.rotation.z = fingerBaseRot[fIdx].z;
|
||||||
|
finger.group.userData.baseRot = {
|
||||||
|
x:finger.group.rotation.x,
|
||||||
|
y:finger.group.rotation.y,
|
||||||
|
z:finger.group.rotation.z
|
||||||
|
};
|
||||||
|
hand.palm.add(finger.group);
|
||||||
|
|
||||||
|
let parent = finger.group;
|
||||||
|
for (let s = 0; s < segLens.length; s++) {
|
||||||
|
const segGroup = new THREE.Group();
|
||||||
|
|
||||||
|
const jGeom = new THREE.SphereGeometry(fingerWidth * 0.6, 8, 8);
|
||||||
|
const joint = new THREE.Mesh(jGeom, jointMaterial);
|
||||||
|
segGroup.add(joint);
|
||||||
|
|
||||||
|
const segGeom = new THREE.BoxGeometry(fingerWidth, fingerHeight, segLens[s]);
|
||||||
|
const seg = new THREE.Mesh(segGeom, fingerMaterial);
|
||||||
|
seg.position.z = -segLens[s] / 2;
|
||||||
|
segGroup.add(seg);
|
||||||
|
|
||||||
|
parent.add(segGroup);
|
||||||
|
|
||||||
|
finger.segments.push(segGroup);
|
||||||
|
finger.joints.push(joint);
|
||||||
|
|
||||||
|
if (s < segLens.length - 1) {
|
||||||
|
const connector = new THREE.Group();
|
||||||
|
connector.position.z = -segLens[s];
|
||||||
|
segGroup.add(connector);
|
||||||
|
parent = connector;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
hand.fingers.push(finger);
|
||||||
|
}
|
||||||
|
|
||||||
|
addFingerLabels();
|
||||||
|
addHandLabel();
|
||||||
|
}
|
||||||
|
|
||||||
|
function addFingerLabels() {
|
||||||
|
const names = ['Thumb','Index','Middle','Ring','Pinky'];
|
||||||
|
for (let i = 0; i < hand.fingers.length; i++) {
|
||||||
|
const finger = hand.fingers[i];
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
canvas.width = 128; canvas.height = 32;
|
||||||
|
ctx.fillStyle = '#ffffff'; ctx.fillRect(0,0,canvas.width,canvas.height);
|
||||||
|
ctx.font = 'bold 16px Arial';
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
|
||||||
|
ctx.fillText(names[i], canvas.width/2, canvas.height/2);
|
||||||
|
|
||||||
|
const texture = new THREE.CanvasTexture(canvas);
|
||||||
|
const geom = new THREE.PlaneGeometry(2, 0.5);
|
||||||
|
const mat = new THREE.MeshBasicMaterial({ map:texture, transparent:true, side:THREE.DoubleSide });
|
||||||
|
const label = new THREE.Mesh(geom, mat);
|
||||||
|
label.position.set(0, -1.5, -2);
|
||||||
|
label.rotation.x = Math.PI / 2;
|
||||||
|
finger.group.add(label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function addHandLabel() {
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
canvas.width = 256; canvas.height = 64;
|
||||||
|
ctx.fillStyle = '#ffffff'; ctx.fillRect(0,0,canvas.width,canvas.height);
|
||||||
|
ctx.font = 'bold 24px Arial';
|
||||||
|
ctx.fillStyle = '#000000';
|
||||||
|
ctx.textAlign = 'center'; ctx.textBaseline = 'middle';
|
||||||
|
ctx.fillText('RIGHT HAND (VERTICAL)', canvas.width/2, canvas.height/2);
|
||||||
|
|
||||||
|
const texture = new THREE.CanvasTexture(canvas);
|
||||||
|
const geom = new THREE.PlaneGeometry(7, 1.75);
|
||||||
|
const mat = new THREE.MeshBasicMaterial({ map:texture, transparent:true, side:THREE.DoubleSide });
|
||||||
|
const label = new THREE.Mesh(geom, mat);
|
||||||
|
label.position.set(0, -2, 0);
|
||||||
|
label.rotation.x = Math.PI / 2;
|
||||||
|
scene.add(label);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateHandModel() {
|
||||||
|
for (let i = 0; i < MAX_JOINTS; i++) {
|
||||||
|
const info = fingerJointMap[i];
|
||||||
|
if (!info) continue;
|
||||||
|
const { finger, joint, type, min, max, angleMin, angleMax } = info;
|
||||||
|
const raw = jointValues[i];
|
||||||
|
const f = hand.fingers[finger];
|
||||||
|
if (!f) continue;
|
||||||
|
|
||||||
|
const center = (min + max) / 2;
|
||||||
|
let angle = 0;
|
||||||
|
|
||||||
|
if (type.includes('ABDUCTION')) {
|
||||||
|
// symmetric around neutral
|
||||||
|
const k = clamp((raw - center) / ((max - min) / 2), -1, 1);
|
||||||
|
angle = angleMin + (k + 1) * 0.5 * (angleMax - angleMin);
|
||||||
|
|
||||||
|
const base = f.group.userData.baseRot || {x:0,y:0,z:0};
|
||||||
|
if (finger === 0 && joint === 0) {
|
||||||
|
// Thumb: abduction about Z (toward/away from palm)
|
||||||
|
f.group.rotation.z = base.z + angle;
|
||||||
|
} else {
|
||||||
|
// Other fingers: side-to-side about Y
|
||||||
|
f.group.rotation.y = base.y + angle;
|
||||||
|
}
|
||||||
|
} else if (type.includes('FLEXION')) {
|
||||||
|
const isThumb = finger === 0;
|
||||||
|
const isMCP = type === 'MCP_FLEXION';
|
||||||
|
const isPIP = type === 'PIP_FLEXION';
|
||||||
|
const positiveOnly = (isThumb && (type === 'MCP_FLEXION' || type === 'IP_FLEXION')) || (!isThumb && isPIP);
|
||||||
|
|
||||||
|
if (positiveOnly) {
|
||||||
|
const t = raw <= center ? 0 : invLerp(center, max, raw); // 0..1
|
||||||
|
angle = angleMin + t * (angleMax - angleMin); // 0..+limit
|
||||||
|
} else {
|
||||||
|
const k = clamp((raw - center) / ((max - min) / 2), -1, 1);
|
||||||
|
angle = angleMin + (k + 1) * 0.5 * (angleMax - angleMin);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isMCP) {
|
||||||
|
// MCP flexion applies to the finger base group (same as abduction)
|
||||||
|
const base = f.group.userData.baseRot || {x:0,y:0,z:0};
|
||||||
|
f.group.rotation.x = base.x + angle;
|
||||||
|
} else if (f.segments[joint]) {
|
||||||
|
// PIP/DIP/IP flexion applies to individual segments
|
||||||
|
f.segments[joint].rotation.x = angle;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Render Loop --------------------
|
||||||
|
function onWindowResize() {
|
||||||
|
camera.aspect = canvasContainer.clientWidth / canvasContainer.clientHeight;
|
||||||
|
camera.updateProjectionMatrix();
|
||||||
|
renderer.setSize(canvasContainer.clientWidth, canvasContainer.clientHeight);
|
||||||
|
}
|
||||||
|
|
||||||
|
function animate() {
|
||||||
|
requestAnimationFrame(animate);
|
||||||
|
controls.update();
|
||||||
|
renderer.render(scene, camera);
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Misc UI --------------------
|
||||||
|
function addLogMessage(msg) {
|
||||||
|
const el = document.createElement('div');
|
||||||
|
el.textContent = msg;
|
||||||
|
logContainer.appendChild(el);
|
||||||
|
logContainer.scrollTop = logContainer.scrollHeight;
|
||||||
|
while (logContainer.children.length > 100) {
|
||||||
|
logContainer.removeChild(logContainer.firstChild);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Camera view controls
|
||||||
|
frontViewBtn?.addEventListener('click', () => { camera.position.set(0, 0, 20); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
sideViewBtn?.addEventListener('click', () => { camera.position.set(20, 0, 0); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
topViewBtn?.addEventListener('click', () => { camera.position.set(0, 20, 0); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
resetViewBtn?.addEventListener('click', () => { camera.position.set(10,10,10); camera.lookAt(0,0,0); controls.update(); });
|
||||||
|
|
||||||
|
// Serial connect buttons
|
||||||
|
connectButton?.addEventListener('click', connectToDevice);
|
||||||
|
disconnectButton?.addEventListener('click', disconnectFromDevice);
|
||||||
|
|
||||||
|
// Web Serial support check
|
||||||
|
if (!navigator.serial) {
|
||||||
|
statusIndicator.textContent = 'Status: Web Serial API not supported in this browser';
|
||||||
|
connectButton.disabled = true;
|
||||||
|
addLogMessage('ERROR: Web Serial API is not supported in this browser. Try Chrome or Edge.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------- Boot --------------------
|
||||||
|
initThreeJS();
|
||||||
|
initializeJointElements();
|
||||||
|
|
||||||
|
// -------------------- Styles (inline) --------------------
|
||||||
|
const styleElement = document.createElement('style');
|
||||||
|
styleElement.textContent = `
|
||||||
|
.joint-info { border-bottom: 1px solid #eee; padding: 8px 0; }
|
||||||
|
.joint-name { font-weight: 600; margin-bottom: 4px; }
|
||||||
|
.joint-value { font-size: 12px; color: #333; margin-bottom: 4px; }
|
||||||
|
.bar-container { width: 100%; height: 8px; background: #ddd; border-radius: 4px; overflow: hidden; }
|
||||||
|
.bar { height: 100%; width: 0%; background: #4caf50; }
|
||||||
|
.joint-slider { width: 100%; margin: 6px 0; }
|
||||||
|
.invert-toggle { display: inline-flex; align-items: center; gap: 6px; margin-top: 4px; font-size: 12px; color: #555; }
|
||||||
|
.limits-row { display: flex; align-items: center; gap: 6px; margin-top: 6px; flex-wrap: wrap; }
|
||||||
|
.limit-label { font-size: 11px; color: #666; }
|
||||||
|
.limit-num { width: 60px; }
|
||||||
|
.calib-row { display: flex; align-items: center; gap: 8px; margin-top: 4px; }
|
||||||
|
.calib-btn { padding: 2px 6px; font-size: 11px; background: #f44336; color: white; border: none; border-radius: 3px; cursor: pointer; }
|
||||||
|
.calib-btn:hover { background: #d32f2f; }
|
||||||
|
.calib-status { font-size: 11px; color: #666; }
|
||||||
|
.status.connected { color: #0a0; }
|
||||||
|
.status.disconnected { color: #a00; }
|
||||||
|
`;
|
||||||
|
document.head.appendChild(styleElement);
|
||||||
Reference in New Issue
Block a user